{"level":"info","ts":"2026-07-13T15:41:04.427Z","logger":"setup","msg":"Loaded feature gates","featureGates":{"RayClusterStatusConditions":true,"RayJobDeletionPolicy":false}}
{"level":"info","ts":"2026-07-13T15:41:04.521Z","logger":"setup","msg":"Flag watchNamespace is not set. Watch custom resources in all namespaces."}
{"level":"info","ts":"2026-07-13T15:41:04.521Z","logger":"setup","msg":"Setup manager"}
{"level":"info","ts":"2026-07-13T15:41:04.723Z","logger":"setup","msg":"NetworkPolicy controller registered (annotation-based activation)"}
{"level":"info","ts":"2026-07-13T15:41:04.723Z","logger":"controller-runtime.builder","msg":"Registering a mutating webhook","GVK":"ray.io/v1, Kind=RayCluster","path":"/mutate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-07-13T15:41:04.723Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/mutate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-07-13T15:41:04.724Z","logger":"controller-runtime.builder","msg":"Registering a validating webhook","GVK":"ray.io/v1, Kind=RayCluster","path":"/validate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-07-13T15:41:04.724Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/validate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-07-13T15:41:04.724Z","logger":"setup","msg":"starting manager"}
{"level":"info","ts":"2026-07-13T15:41:04.724Z","logger":"controller-runtime.metrics","msg":"Starting metrics server"}
{"level":"info","ts":"2026-07-13T15:41:04.724Z","msg":"starting server","name":"health probe","addr":"[::]:8082"}
{"level":"info","ts":"2026-07-13T15:41:04.724Z","logger":"controller-runtime.webhook","msg":"Starting webhook server"}
{"level":"info","ts":"2026-07-13T15:41:04.724Z","logger":"controller-runtime.metrics","msg":"Serving metrics server","bindAddress":":8080","secure":false}
{"level":"info","ts":"2026-07-13T15:41:04.724Z","logger":"controller-runtime.certwatcher","msg":"Updated current TLS certificate","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key"}
{"level":"info","ts":"2026-07-13T15:41:04.724Z","logger":"controller-runtime.certwatcher","msg":"Starting certificate poll+watcher","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key","interval":10}
{"level":"info","ts":"2026-07-13T15:41:04.724Z","logger":"controller-runtime.webhook","msg":"Serving webhook server","host":"","port":9443}
{"level":"info","ts":"2026-07-13T15:41:04.924Z","msg":"attempting to acquire leader lease opendatahub/ray-operator-leader..."}
{"level":"info","ts":"2026-07-13T15:41:04.931Z","msg":"successfully acquired lease opendatahub/ray-operator-leader"}
{"level":"info","ts":"2026-07-13T15:41:04.931Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-07-13T15:41:04.931Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.Pod"}
{"level":"info","ts":"2026-07-13T15:41:04.931Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-07-13T15:41:04.931Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-07-13T15:41:04.931Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.RayService"}
{"level":"info","ts":"2026-07-13T15:41:04.931Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-07-13T15:41:04.932Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-07-13T15:41:04.932Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.Route"}
{"level":"info","ts":"2026-07-13T15:41:04.932Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.ServiceAccount"}
{"level":"info","ts":"2026-07-13T15:41:04.932Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.RayJob"}
{"level":"info","ts":"2026-07-13T15:41:04.932Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-07-13T15:41:04.932Z","msg":"Starting EventSource","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-07-13T15:41:04.932Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.Job"}
{"level":"info","ts":"2026-07-13T15:41:04.932Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-07-13T15:41:04.932Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-07-13T15:41:04.932Z","msg":"Starting EventSource","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.NetworkPolicy"}
{"level":"info","ts":"2026-07-13T15:41:04.932Z","msg":"Starting EventSource","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-07-13T15:41:05.423Z","msg":"Starting Controller","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster"}
{"level":"info","ts":"2026-07-13T15:41:05.423Z","msg":"Starting workers","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","worker count":1}
{"level":"info","ts":"2026-07-13T15:41:05.433Z","logger":"controllers.RayCluster","msg":"Starting Controller"}
{"level":"info","ts":"2026-07-13T15:41:05.433Z","logger":"controllers.RayService","msg":"Starting Controller"}
{"level":"info","ts":"2026-07-13T15:41:05.433Z","logger":"controllers.RayService","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-07-13T15:41:05.433Z","logger":"controllers.RayCluster","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-07-13T15:41:05.522Z","msg":"Starting Controller","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster"}
{"level":"info","ts":"2026-07-13T15:41:05.522Z","msg":"Starting workers","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","worker count":1}
{"level":"info","ts":"2026-07-13T15:41:05.522Z","msg":"Starting Controller","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster"}
{"level":"info","ts":"2026-07-13T15:41:05.522Z","msg":"Starting workers","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","worker count":1}
{"level":"info","ts":"2026-07-13T15:41:05.522Z","logger":"controllers.RayJob","msg":"Starting Controller"}
{"level":"info","ts":"2026-07-13T15:41:05.522Z","logger":"controllers.RayJob","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-07-13T15:43:24.263Z","logger":"raycluster-resource","msg":"default","name":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:24.263Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster-auth","namespace":"test-ns-k9mkf"}
{"level":"info","ts":"2026-07-13T15:43:24.430Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a33c4ccb-aeaa-4c57-9f38-a1af56d80796","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:24.430Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"557f43f1-6f80-4700-9263-c441e6a57451","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"}}
{"level":"info","ts":"2026-07-13T15:43:24.430Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"7b9211cf-00d2-48aa-a9da-20817ca3796d","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:24.430Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"557f43f1-6f80-4700-9263-c441e6a57451","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:24.430Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"557f43f1-6f80-4700-9263-c441e6a57451","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:24.430Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1659b3de-8640-43ef-8841-eff25ded7e3e"}
{"level":"info","ts":"2026-07-13T15:43:24.430Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1659b3de-8640-43ef-8841-eff25ded7e3e"}
{"level":"info","ts":"2026-07-13T15:43:24.430Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"557f43f1-6f80-4700-9263-c441e6a57451","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:24.436Z","logger":"raycluster-resource","msg":"default","name":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:24.436Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster-auth","namespace":"test-ns-k9mkf"}
{"level":"info","ts":"2026-07-13T15:43:24.521Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"557f43f1-6f80-4700-9263-c441e6a57451","name":"raycluster-auth-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:43:24.521Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1659b3de-8640-43ef-8841-eff25ded7e3e","name":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:43:24.529Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"7b9211cf-00d2-48aa-a9da-20817ca3796d","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:24.529Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"7447f45d-d1e5-49fe-ac5d-b722f1f6cc62","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:24.530Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a33c4ccb-aeaa-4c57-9f38-a1af56d80796","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:24.622Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a33c4ccb-aeaa-4c57-9f38-a1af56d80796","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:24.626Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"7447f45d-d1e5-49fe-ac5d-b722f1f6cc62","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-13T15:43:24.627Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"557f43f1-6f80-4700-9263-c441e6a57451","name":"kuberay-gateway-access","namespace":"test-ns-k9mkf","operation":"created"}
{"level":"info","ts":"2026-07-13T15:43:24.722Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a33c4ccb-aeaa-4c57-9f38-a1af56d80796","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:24.821Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"7447f45d-d1e5-49fe-ac5d-b722f1f6cc62","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-13T15:43:24.821Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"7447f45d-d1e5-49fe-ac5d-b722f1f6cc62","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:24.821Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"4b564bf7-b8c8-49ae-ba6f-45b06024dc17","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:24.821Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a33c4ccb-aeaa-4c57-9f38-a1af56d80796","rayCluster":"raycluster-auth","secret":"ca-secret"}
{"level":"info","ts":"2026-07-13T15:43:24.821Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a33c4ccb-aeaa-4c57-9f38-a1af56d80796","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:24.921Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a33c4ccb-aeaa-4c57-9f38-a1af56d80796","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:24.926Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"4b564bf7-b8c8-49ae-ba6f-45b06024dc17","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-13T15:43:25.022Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a33c4ccb-aeaa-4c57-9f38-a1af56d80796","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:43:25.022Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"557f43f1-6f80-4700-9263-c441e6a57451","name":"test-ns-k9mkf-raycluster-auth","operation":"created"}
{"level":"info","ts":"2026-07-13T15:43:25.026Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"4b564bf7-b8c8-49ae-ba6f-45b06024dc17","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-13T15:43:25.026Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"4b564bf7-b8c8-49ae-ba6f-45b06024dc17","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:25.122Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a33c4ccb-aeaa-4c57-9f38-a1af56d80796","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"error","ts":"2026-07-13T15:43:25.122Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a33c4ccb-aeaa-4c57-9f38-a1af56d80796","error":"Certificate.cert-manager.io \"ray-worker-cert-raycluster-auth\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:25.122Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a33c4ccb-aeaa-4c57-9f38-a1af56d80796"}
{"level":"error","ts":"2026-07-13T15:43:25.122Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a33c4ccb-aeaa-4c57-9f38-a1af56d80796","error":"Certificate.cert-manager.io \"ray-worker-cert-raycluster-auth\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:25.122Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a1cde6ce-6bb3-4708-90d6-7430a6ef5637","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:25.122Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a1cde6ce-6bb3-4708-90d6-7430a6ef5637"}
{"level":"info","ts":"2026-07-13T15:43:25.127Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e5c15c33-50d8-43a6-8545-c8791e4024bf","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:25.127Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e5c15c33-50d8-43a6-8545-c8791e4024bf"}
{"level":"info","ts":"2026-07-13T15:43:25.222Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1659b3de-8640-43ef-8841-eff25ded7e3e","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-07-13T15:43:25.222Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1659b3de-8640-43ef-8841-eff25ded7e3e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:25.222Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1659b3de-8640-43ef-8841-eff25ded7e3e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-07-13T15:43:25.222Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1659b3de-8640-43ef-8841-eff25ded7e3e","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:25.222Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1659b3de-8640-43ef-8841-eff25ded7e3e","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:25.222Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1659b3de-8640-43ef-8841-eff25ded7e3e","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:25.222Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1659b3de-8640-43ef-8841-eff25ded7e3e","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:43:25.222Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1659b3de-8640-43ef-8841-eff25ded7e3e","name":"raycluster-auth","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:43:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:43:25.228Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1659b3de-8640-43ef-8841-eff25ded7e3e","name":"raycluster-auth","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-k9mkf","name":"raycluster-auth"}}
{"level":"info","ts":"2026-07-13T15:43:25.228Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1659b3de-8640-43ef-8841-eff25ded7e3e"}
{"level":"error","ts":"2026-07-13T15:43:25.228Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1659b3de-8640-43ef-8841-eff25ded7e3e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:25.228Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68508c0f-3c2e-439a-96eb-5c5398074fb0"}
{"level":"info","ts":"2026-07-13T15:43:25.228Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68508c0f-3c2e-439a-96eb-5c5398074fb0"}
{"level":"info","ts":"2026-07-13T15:43:25.228Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68508c0f-3c2e-439a-96eb-5c5398074fb0","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:43:25.228Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68508c0f-3c2e-439a-96eb-5c5398074fb0","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-07-13T15:43:25.228Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68508c0f-3c2e-439a-96eb-5c5398074fb0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:25.228Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68508c0f-3c2e-439a-96eb-5c5398074fb0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-07-13T15:43:25.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68508c0f-3c2e-439a-96eb-5c5398074fb0","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:25.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68508c0f-3c2e-439a-96eb-5c5398074fb0","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:25.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68508c0f-3c2e-439a-96eb-5c5398074fb0","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:25.228Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68508c0f-3c2e-439a-96eb-5c5398074fb0","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:43:25.228Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68508c0f-3c2e-439a-96eb-5c5398074fb0","name":"raycluster-auth","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:43:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:43:25.421Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68508c0f-3c2e-439a-96eb-5c5398074fb0"}
{"level":"error","ts":"2026-07-13T15:43:25.421Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68508c0f-3c2e-439a-96eb-5c5398074fb0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:25.421Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"acae0640-c489-48d5-8f30-cb65ca112181","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:25.421Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"69b2d2ca-0bab-42b8-a332-8e8bc444e9eb"}
{"level":"info","ts":"2026-07-13T15:43:25.421Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"69b2d2ca-0bab-42b8-a332-8e8bc444e9eb"}
{"level":"info","ts":"2026-07-13T15:43:25.421Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"69b2d2ca-0bab-42b8-a332-8e8bc444e9eb","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:43:25.421Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"8f9b7580-68c6-4459-a894-6aa0cc484770","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:25.422Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"69b2d2ca-0bab-42b8-a332-8e8bc444e9eb","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-07-13T15:43:25.422Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"69b2d2ca-0bab-42b8-a332-8e8bc444e9eb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:25.422Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"69b2d2ca-0bab-42b8-a332-8e8bc444e9eb","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-07-13T15:43:25.422Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"69b2d2ca-0bab-42b8-a332-8e8bc444e9eb","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:25.422Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"8f9b7580-68c6-4459-a894-6aa0cc484770"}
{"level":"info","ts":"2026-07-13T15:43:25.422Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"69b2d2ca-0bab-42b8-a332-8e8bc444e9eb","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:25.422Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"69b2d2ca-0bab-42b8-a332-8e8bc444e9eb","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:25.422Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"69b2d2ca-0bab-42b8-a332-8e8bc444e9eb"}
{"level":"error","ts":"2026-07-13T15:43:25.422Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"69b2d2ca-0bab-42b8-a332-8e8bc444e9eb","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:25.521Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"a6bf2ffa-fcab-47dd-b859-0924f346ae33"}
{"level":"info","ts":"2026-07-13T15:43:25.521Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"a6bf2ffa-fcab-47dd-b859-0924f346ae33"}
{"level":"info","ts":"2026-07-13T15:43:25.521Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"a6bf2ffa-fcab-47dd-b859-0924f346ae33","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:43:25.521Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"a6bf2ffa-fcab-47dd-b859-0924f346ae33","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-07-13T15:43:25.521Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"a6bf2ffa-fcab-47dd-b859-0924f346ae33","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:25.521Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"a6bf2ffa-fcab-47dd-b859-0924f346ae33","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-07-13T15:43:25.521Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"a6bf2ffa-fcab-47dd-b859-0924f346ae33","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:25.521Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"a6bf2ffa-fcab-47dd-b859-0924f346ae33","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:25.522Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"a6bf2ffa-fcab-47dd-b859-0924f346ae33","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:25.522Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"a6bf2ffa-fcab-47dd-b859-0924f346ae33"}
{"level":"error","ts":"2026-07-13T15:43:25.522Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"a6bf2ffa-fcab-47dd-b859-0924f346ae33","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:25.530Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"acae0640-c489-48d5-8f30-cb65ca112181","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-13T15:43:25.621Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4c2777c3-e4a8-4d09-bb4b-f8ceeac9c422"}
{"level":"info","ts":"2026-07-13T15:43:25.621Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4c2777c3-e4a8-4d09-bb4b-f8ceeac9c422"}
{"level":"info","ts":"2026-07-13T15:43:25.621Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4c2777c3-e4a8-4d09-bb4b-f8ceeac9c422","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:43:25.621Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4c2777c3-e4a8-4d09-bb4b-f8ceeac9c422","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-07-13T15:43:25.621Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4c2777c3-e4a8-4d09-bb4b-f8ceeac9c422","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:25.621Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4c2777c3-e4a8-4d09-bb4b-f8ceeac9c422","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-07-13T15:43:25.621Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4c2777c3-e4a8-4d09-bb4b-f8ceeac9c422","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:25.621Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4c2777c3-e4a8-4d09-bb4b-f8ceeac9c422","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:25.621Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4c2777c3-e4a8-4d09-bb4b-f8ceeac9c422","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:25.622Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4c2777c3-e4a8-4d09-bb4b-f8ceeac9c422"}
{"level":"error","ts":"2026-07-13T15:43:25.622Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4c2777c3-e4a8-4d09-bb4b-f8ceeac9c422","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:25.625Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"acae0640-c489-48d5-8f30-cb65ca112181","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-13T15:43:25.625Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"acae0640-c489-48d5-8f30-cb65ca112181","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:25.721Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"81966c6e-3aa7-440c-a30f-7d5fee779cd6"}
{"level":"info","ts":"2026-07-13T15:43:25.721Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"81966c6e-3aa7-440c-a30f-7d5fee779cd6"}
{"level":"info","ts":"2026-07-13T15:43:25.721Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"81966c6e-3aa7-440c-a30f-7d5fee779cd6","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:43:25.721Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"81966c6e-3aa7-440c-a30f-7d5fee779cd6","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-07-13T15:43:25.721Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"81966c6e-3aa7-440c-a30f-7d5fee779cd6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:25.721Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"81966c6e-3aa7-440c-a30f-7d5fee779cd6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-07-13T15:43:25.721Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"81966c6e-3aa7-440c-a30f-7d5fee779cd6","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:25.721Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"81966c6e-3aa7-440c-a30f-7d5fee779cd6","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:25.721Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"81966c6e-3aa7-440c-a30f-7d5fee779cd6","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:25.721Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"81966c6e-3aa7-440c-a30f-7d5fee779cd6"}
{"level":"error","ts":"2026-07-13T15:43:25.722Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"81966c6e-3aa7-440c-a30f-7d5fee779cd6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:25.882Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"2889e1d1-ac96-44f8-99dc-33e61b7d2ab1"}
{"level":"info","ts":"2026-07-13T15:43:25.882Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"2889e1d1-ac96-44f8-99dc-33e61b7d2ab1"}
{"level":"info","ts":"2026-07-13T15:43:25.882Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"2889e1d1-ac96-44f8-99dc-33e61b7d2ab1","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:43:25.882Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"2889e1d1-ac96-44f8-99dc-33e61b7d2ab1","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-07-13T15:43:25.882Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"2889e1d1-ac96-44f8-99dc-33e61b7d2ab1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:25.882Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"2889e1d1-ac96-44f8-99dc-33e61b7d2ab1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-07-13T15:43:25.882Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"2889e1d1-ac96-44f8-99dc-33e61b7d2ab1","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:25.882Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"2889e1d1-ac96-44f8-99dc-33e61b7d2ab1","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:25.882Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"2889e1d1-ac96-44f8-99dc-33e61b7d2ab1","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:25.883Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"2889e1d1-ac96-44f8-99dc-33e61b7d2ab1"}
{"level":"error","ts":"2026-07-13T15:43:25.883Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"2889e1d1-ac96-44f8-99dc-33e61b7d2ab1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:26.221Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"373a9d6f-6d85-4947-8ae8-f7591e5a6a9c"}
{"level":"info","ts":"2026-07-13T15:43:26.221Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"373a9d6f-6d85-4947-8ae8-f7591e5a6a9c"}
{"level":"info","ts":"2026-07-13T15:43:26.221Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"373a9d6f-6d85-4947-8ae8-f7591e5a6a9c","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:43:26.221Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"373a9d6f-6d85-4947-8ae8-f7591e5a6a9c"}
{"level":"info","ts":"2026-07-13T15:43:26.221Z","logger":"controllers.RayCluster","msg":"Waiting for authentication resources to be ready","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"373a9d6f-6d85-4947-8ae8-f7591e5a6a9c","condition":"nil","reason":"AuthenticationPending","conditionStale":false,"observedGeneration":0,"currentGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"error","ts":"2026-07-13T15:43:26.221Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"373a9d6f-6d85-4947-8ae8-f7591e5a6a9c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Waiting for AuthenticationController to create authentication resources","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:26.221Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"373a9d6f-6d85-4947-8ae8-f7591e5a6a9c","error":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Waiting for AuthenticationController to create authentication resources"}
{"level":"info","ts":"2026-07-13T15:43:26.222Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"373a9d6f-6d85-4947-8ae8-f7591e5a6a9c","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:26.222Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"373a9d6f-6d85-4947-8ae8-f7591e5a6a9c","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:26.222Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"373a9d6f-6d85-4947-8ae8-f7591e5a6a9c","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:43:26.222Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"373a9d6f-6d85-4947-8ae8-f7591e5a6a9c","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T15:43:26Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Waiting for AuthenticationController to create authentication resources"}]}
{"level":"info","ts":"2026-07-13T15:43:26.222Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"373a9d6f-6d85-4947-8ae8-f7591e5a6a9c","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:43:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:43:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T15:43:26Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Waiting for AuthenticationController to create authentication resources"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:43:26.334Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"373a9d6f-6d85-4947-8ae8-f7591e5a6a9c"}
{"level":"error","ts":"2026-07-13T15:43:26.334Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"373a9d6f-6d85-4947-8ae8-f7591e5a6a9c","error":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Waiting for AuthenticationController to create authentication resources","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:26.334Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"4bb22a67-269a-4105-b8a0-9c300b7450e4","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:26.334Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e9a0dc4e-2142-4c2d-bbd3-1ab777fa6d9b","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:26.334Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"4bb22a67-269a-4105-b8a0-9c300b7450e4"}
{"level":"info","ts":"2026-07-13T15:43:26.346Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e9a0dc4e-2142-4c2d-bbd3-1ab777fa6d9b","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-13T15:43:26.426Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e9a0dc4e-2142-4c2d-bbd3-1ab777fa6d9b","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-13T15:43:26.426Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e9a0dc4e-2142-4c2d-bbd3-1ab777fa6d9b","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:26.427Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"557f43f1-6f80-4700-9263-c441e6a57451","name":"kube-rbac-proxy-config-raycluster-auth","operation":"created"}
{"level":"error","ts":"2026-07-13T15:43:26.433Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"557f43f1-6f80-4700-9263-c441e6a57451","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:26.433Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"557f43f1-6f80-4700-9263-c441e6a57451"}
{"level":"error","ts":"2026-07-13T15:43:26.433Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"557f43f1-6f80-4700-9263-c441e6a57451","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:26.433Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"cd30609c-1d10-40c8-9193-423dfea788e0","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"}}
{"level":"info","ts":"2026-07-13T15:43:26.433Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"cd30609c-1d10-40c8-9193-423dfea788e0","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:26.433Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"cd30609c-1d10-40c8-9193-423dfea788e0","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:26.445Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"cd30609c-1d10-40c8-9193-423dfea788e0","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:26.445Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"51f9631c-642a-4b75-80d6-ff1b8a32e2f1","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"}}
{"level":"info","ts":"2026-07-13T15:43:26.445Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"51f9631c-642a-4b75-80d6-ff1b8a32e2f1","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:26.445Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"51f9631c-642a-4b75-80d6-ff1b8a32e2f1","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:26.446Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"013c2ed5-84c5-4985-9649-4290064a2709","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:26.446Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"210dcf97-76be-4920-8a86-df683240d138","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:26.446Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"013c2ed5-84c5-4985-9649-4290064a2709"}
{"level":"error","ts":"2026-07-13T15:43:26.455Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"51f9631c-642a-4b75-80d6-ff1b8a32e2f1","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:26.455Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"51f9631c-642a-4b75-80d6-ff1b8a32e2f1"}
{"level":"error","ts":"2026-07-13T15:43:26.455Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"51f9631c-642a-4b75-80d6-ff1b8a32e2f1","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:26.521Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"11e2358e-a263-4f01-b4f1-ea6d16cfe603","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"}}
{"level":"info","ts":"2026-07-13T15:43:26.521Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"11e2358e-a263-4f01-b4f1-ea6d16cfe603","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:26.521Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"11e2358e-a263-4f01-b4f1-ea6d16cfe603","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:26.521Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"210dcf97-76be-4920-8a86-df683240d138","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-13T15:43:26.530Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"11e2358e-a263-4f01-b4f1-ea6d16cfe603","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:26.533Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"210dcf97-76be-4920-8a86-df683240d138","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-13T15:43:26.533Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"210dcf97-76be-4920-8a86-df683240d138","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:26.975Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a"}
{"level":"info","ts":"2026-07-13T15:43:26.975Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a"}
{"level":"info","ts":"2026-07-13T15:43:26.975Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:43:26.975Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a"}
{"level":"info","ts":"2026-07-13T15:43:26.975Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:43:26.975Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a"}
{"level":"info","ts":"2026-07-13T15:43:26.975Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:26.975Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:43:26.975Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T15:43:26.975Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T15:43:26.975Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T15:43:26.975Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:43:26.975Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:43:26.975Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","enabled":true}
{"level":"info","ts":"2026-07-13T15:43:26.993Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","name":"raycluster-auth-head-czcms"}
{"level":"info","ts":"2026-07-13T15:43:26.993Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:43:26.993Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:43:26.993Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T15:43:26.993Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T15:43:26.993Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T15:43:26.993Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a"}
{"level":"info","ts":"2026-07-13T15:43:26.993Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T15:43:26.993Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:43:26.993Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:43:26.993Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","enabled":true}
{"level":"info","ts":"2026-07-13T15:43:27.008Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","name":"raycluster-auth-small-group-worker-z4nbk"}
{"level":"info","ts":"2026-07-13T15:43:27.008Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:43:27.009Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:43:27.009Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"raycluster-auth-head-svc"},"newHead":{"podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"}}
{"level":"info","ts":"2026-07-13T15:43:27.009Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"8c316b7c-8960-46f7-a348-ca7569c3575a","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:43:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T15:43:26Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Waiting for AuthenticationController to create authentication resources"},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:43:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:43:27.026Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"034ecb36-eabc-419d-96b4-5606829e2104"}
{"level":"info","ts":"2026-07-13T15:43:27.026Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"034ecb36-eabc-419d-96b4-5606829e2104"}
{"level":"info","ts":"2026-07-13T15:43:27.026Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"034ecb36-eabc-419d-96b4-5606829e2104","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:43:27.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"034ecb36-eabc-419d-96b4-5606829e2104","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:43:27.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"034ecb36-eabc-419d-96b4-5606829e2104","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:43:27.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"034ecb36-eabc-419d-96b4-5606829e2104","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:43:27.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"034ecb36-eabc-419d-96b4-5606829e2104","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:43:27.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"034ecb36-eabc-419d-96b4-5606829e2104","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:43:27.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"034ecb36-eabc-419d-96b4-5606829e2104","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:43:27.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"034ecb36-eabc-419d-96b4-5606829e2104","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:43:27.026Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"034ecb36-eabc-419d-96b4-5606829e2104","podName":"raycluster-auth-small-group-worker-z4nbk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:43:27.026Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"034ecb36-eabc-419d-96b4-5606829e2104","podName":"raycluster-auth-small-group-worker-z4nbk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:43:27.026Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"fd2bd9a8-6b4b-4d62-89b3-5098a413762c","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:27.026Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"034ecb36-eabc-419d-96b4-5606829e2104","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"raycluster-auth-head-svc"},"newHead":{"podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"}}
{"level":"info","ts":"2026-07-13T15:43:27.026Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"5243e083-afe6-4e44-97b4-659e8c91820a","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:27.026Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"034ecb36-eabc-419d-96b4-5606829e2104","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:43:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T15:43:26Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Waiting for AuthenticationController to create authentication resources"},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:43:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:43:27.026Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"fd2bd9a8-6b4b-4d62-89b3-5098a413762c","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:43:27.034Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"034ecb36-eabc-419d-96b4-5606829e2104","name":"raycluster-auth","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-k9mkf","name":"raycluster-auth"}}
{"level":"info","ts":"2026-07-13T15:43:27.034Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"034ecb36-eabc-419d-96b4-5606829e2104"}
{"level":"error","ts":"2026-07-13T15:43:27.034Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"034ecb36-eabc-419d-96b4-5606829e2104","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:27.034Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0686e65-c68c-4ba5-b103-1472d0500408"}
{"level":"info","ts":"2026-07-13T15:43:27.034Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0686e65-c68c-4ba5-b103-1472d0500408"}
{"level":"info","ts":"2026-07-13T15:43:27.034Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0686e65-c68c-4ba5-b103-1472d0500408","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:43:27.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0686e65-c68c-4ba5-b103-1472d0500408","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:43:27.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0686e65-c68c-4ba5-b103-1472d0500408","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:43:27.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0686e65-c68c-4ba5-b103-1472d0500408","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:43:27.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0686e65-c68c-4ba5-b103-1472d0500408","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:43:27.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0686e65-c68c-4ba5-b103-1472d0500408","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:43:27.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0686e65-c68c-4ba5-b103-1472d0500408","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:43:27.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0686e65-c68c-4ba5-b103-1472d0500408","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:43:27.034Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0686e65-c68c-4ba5-b103-1472d0500408","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:43:27.034Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0686e65-c68c-4ba5-b103-1472d0500408","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:43:27.034Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0686e65-c68c-4ba5-b103-1472d0500408","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:43:27.034Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0686e65-c68c-4ba5-b103-1472d0500408","seconds":300}
{"level":"info","ts":"2026-07-13T15:43:27.039Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"5243e083-afe6-4e44-97b4-659e8c91820a","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-13T15:43:27.039Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c2efd190-a940-4c2f-9230-1ff204153a00"}
{"level":"info","ts":"2026-07-13T15:43:27.039Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c2efd190-a940-4c2f-9230-1ff204153a00"}
{"level":"info","ts":"2026-07-13T15:43:27.039Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c2efd190-a940-4c2f-9230-1ff204153a00","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:43:27.039Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c2efd190-a940-4c2f-9230-1ff204153a00","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:43:27.039Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c2efd190-a940-4c2f-9230-1ff204153a00","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:43:27.039Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c2efd190-a940-4c2f-9230-1ff204153a00","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:43:27.039Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c2efd190-a940-4c2f-9230-1ff204153a00","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:43:27.039Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c2efd190-a940-4c2f-9230-1ff204153a00","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:43:27.039Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c2efd190-a940-4c2f-9230-1ff204153a00","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:43:27.039Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c2efd190-a940-4c2f-9230-1ff204153a00","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:43:27.039Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c2efd190-a940-4c2f-9230-1ff204153a00","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:43:27.039Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c2efd190-a940-4c2f-9230-1ff204153a00","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:43:27.039Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c2efd190-a940-4c2f-9230-1ff204153a00","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:43:27.039Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c2efd190-a940-4c2f-9230-1ff204153a00","seconds":300}
{"level":"info","ts":"2026-07-13T15:43:27.045Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6872eaa2-ca6e-45bc-98fe-2b4947279e06"}
{"level":"info","ts":"2026-07-13T15:43:27.045Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6872eaa2-ca6e-45bc-98fe-2b4947279e06"}
{"level":"info","ts":"2026-07-13T15:43:27.045Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6872eaa2-ca6e-45bc-98fe-2b4947279e06","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:43:27.045Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6872eaa2-ca6e-45bc-98fe-2b4947279e06","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:43:27.045Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6872eaa2-ca6e-45bc-98fe-2b4947279e06","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:43:27.045Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6872eaa2-ca6e-45bc-98fe-2b4947279e06","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:43:27.045Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6872eaa2-ca6e-45bc-98fe-2b4947279e06","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:43:27.045Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6872eaa2-ca6e-45bc-98fe-2b4947279e06","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:43:27.045Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6872eaa2-ca6e-45bc-98fe-2b4947279e06","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:43:27.045Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6872eaa2-ca6e-45bc-98fe-2b4947279e06","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:43:27.045Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6872eaa2-ca6e-45bc-98fe-2b4947279e06","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:43:27.045Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6872eaa2-ca6e-45bc-98fe-2b4947279e06","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:43:27.045Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6872eaa2-ca6e-45bc-98fe-2b4947279e06","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:43:27.045Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6872eaa2-ca6e-45bc-98fe-2b4947279e06","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:43:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:43:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:43:27.050Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"5243e083-afe6-4e44-97b4-659e8c91820a","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-13T15:43:27.050Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"5243e083-afe6-4e44-97b4-659e8c91820a","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:27.064Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"05093238-3ca3-4979-8100-92bfcfb65106"}
{"level":"info","ts":"2026-07-13T15:43:27.064Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"05093238-3ca3-4979-8100-92bfcfb65106"}
{"level":"info","ts":"2026-07-13T15:43:27.064Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"05093238-3ca3-4979-8100-92bfcfb65106","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:43:27.064Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"05093238-3ca3-4979-8100-92bfcfb65106","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:43:27.064Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"05093238-3ca3-4979-8100-92bfcfb65106","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:43:27.064Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"05093238-3ca3-4979-8100-92bfcfb65106","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:43:27.064Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"05093238-3ca3-4979-8100-92bfcfb65106","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:43:27.064Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"05093238-3ca3-4979-8100-92bfcfb65106","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:43:27.064Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"05093238-3ca3-4979-8100-92bfcfb65106","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:43:27.064Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"05093238-3ca3-4979-8100-92bfcfb65106","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:43:27.064Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"05093238-3ca3-4979-8100-92bfcfb65106","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:43:27.064Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"05093238-3ca3-4979-8100-92bfcfb65106","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:43:27.064Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"05093238-3ca3-4979-8100-92bfcfb65106","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:43:27.064Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"38a7436c-4539-4f48-b9cc-17ec773a204f","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:27.064Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"95119d92-f6c9-4c20-b018-f6685c1e5f6f","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:27.064Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"05093238-3ca3-4979-8100-92bfcfb65106","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:43:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:43:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:43:27.064Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"95119d92-f6c9-4c20-b018-f6685c1e5f6f","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:43:27.073Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"05093238-3ca3-4979-8100-92bfcfb65106","name":"raycluster-auth","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-k9mkf","name":"raycluster-auth"}}
{"level":"info","ts":"2026-07-13T15:43:27.073Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"05093238-3ca3-4979-8100-92bfcfb65106"}
{"level":"error","ts":"2026-07-13T15:43:27.073Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"05093238-3ca3-4979-8100-92bfcfb65106","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:43:27.078Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"fbee5dd5-0751-4dd6-b11f-be82d1fc3800"}
{"level":"info","ts":"2026-07-13T15:43:27.078Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"fbee5dd5-0751-4dd6-b11f-be82d1fc3800"}
{"level":"info","ts":"2026-07-13T15:43:27.078Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"fbee5dd5-0751-4dd6-b11f-be82d1fc3800","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:43:27.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"fbee5dd5-0751-4dd6-b11f-be82d1fc3800","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:43:27.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"fbee5dd5-0751-4dd6-b11f-be82d1fc3800","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:43:27.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"fbee5dd5-0751-4dd6-b11f-be82d1fc3800","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:43:27.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"fbee5dd5-0751-4dd6-b11f-be82d1fc3800","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:43:27.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"fbee5dd5-0751-4dd6-b11f-be82d1fc3800","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:43:27.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"fbee5dd5-0751-4dd6-b11f-be82d1fc3800","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:43:27.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"fbee5dd5-0751-4dd6-b11f-be82d1fc3800","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:43:27.078Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"fbee5dd5-0751-4dd6-b11f-be82d1fc3800","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:43:27.078Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"fbee5dd5-0751-4dd6-b11f-be82d1fc3800","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:43:27.078Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"fbee5dd5-0751-4dd6-b11f-be82d1fc3800","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:43:27.078Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"fbee5dd5-0751-4dd6-b11f-be82d1fc3800","seconds":300}
{"level":"info","ts":"2026-07-13T15:43:27.126Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"38a7436c-4539-4f48-b9cc-17ec773a204f","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-13T15:43:27.135Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"38a7436c-4539-4f48-b9cc-17ec773a204f","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-13T15:43:27.135Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"38a7436c-4539-4f48-b9cc-17ec773a204f","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:27.442Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68b21dfb-8ff8-471f-814a-5d0bdd37f6f8"}
{"level":"info","ts":"2026-07-13T15:43:27.442Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68b21dfb-8ff8-471f-814a-5d0bdd37f6f8"}
{"level":"info","ts":"2026-07-13T15:43:27.442Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68b21dfb-8ff8-471f-814a-5d0bdd37f6f8","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:43:27.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68b21dfb-8ff8-471f-814a-5d0bdd37f6f8","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:43:27.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68b21dfb-8ff8-471f-814a-5d0bdd37f6f8","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:43:27.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68b21dfb-8ff8-471f-814a-5d0bdd37f6f8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:43:27.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68b21dfb-8ff8-471f-814a-5d0bdd37f6f8","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:43:27.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68b21dfb-8ff8-471f-814a-5d0bdd37f6f8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:43:27.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68b21dfb-8ff8-471f-814a-5d0bdd37f6f8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:43:27.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68b21dfb-8ff8-471f-814a-5d0bdd37f6f8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:43:27.442Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68b21dfb-8ff8-471f-814a-5d0bdd37f6f8","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:43:27.442Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68b21dfb-8ff8-471f-814a-5d0bdd37f6f8","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:43:27.442Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68b21dfb-8ff8-471f-814a-5d0bdd37f6f8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:43:27.443Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"68b21dfb-8ff8-471f-814a-5d0bdd37f6f8","seconds":300}
{"level":"info","ts":"2026-07-13T15:43:27.462Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3909afac-0f9e-4795-9022-04add6c48e39"}
{"level":"info","ts":"2026-07-13T15:43:27.462Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3909afac-0f9e-4795-9022-04add6c48e39"}
{"level":"info","ts":"2026-07-13T15:43:27.462Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3909afac-0f9e-4795-9022-04add6c48e39","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:43:27.462Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3909afac-0f9e-4795-9022-04add6c48e39","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:43:27.462Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3909afac-0f9e-4795-9022-04add6c48e39","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:43:27.462Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3909afac-0f9e-4795-9022-04add6c48e39","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:43:27.462Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3909afac-0f9e-4795-9022-04add6c48e39","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:43:27.462Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3909afac-0f9e-4795-9022-04add6c48e39","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:43:27.462Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3909afac-0f9e-4795-9022-04add6c48e39","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:43:27.462Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3909afac-0f9e-4795-9022-04add6c48e39","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:43:27.463Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3909afac-0f9e-4795-9022-04add6c48e39","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:43:27.463Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3909afac-0f9e-4795-9022-04add6c48e39","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:43:27.463Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3909afac-0f9e-4795-9022-04add6c48e39","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:43:27.463Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3909afac-0f9e-4795-9022-04add6c48e39","seconds":300}
{"level":"info","ts":"2026-07-13T15:43:55.128Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"712f0c5e-5aee-4020-b228-4b65071e0541","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:43:55.128Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"712f0c5e-5aee-4020-b228-4b65071e0541","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:44:55.129Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"c091c624-c8d8-428e-9191-98fe07f85502","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:44:55.129Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"c091c624-c8d8-428e-9191-98fe07f85502","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:45:55.130Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"57116dd2-2b7d-4b78-be6a-9831ed23153d","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:45:55.130Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"57116dd2-2b7d-4b78-be6a-9831ed23153d","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:46:55.131Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"01d3e714-012e-4a99-95c3-c13e3556d1c0","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:46:55.132Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"01d3e714-012e-4a99-95c3-c13e3556d1c0","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:47:20.303Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6c6850f6-15f0-451f-b1d2-250379444f6a"}
{"level":"info","ts":"2026-07-13T15:47:20.303Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6c6850f6-15f0-451f-b1d2-250379444f6a"}
{"level":"info","ts":"2026-07-13T15:47:20.303Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6c6850f6-15f0-451f-b1d2-250379444f6a","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:47:20.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6c6850f6-15f0-451f-b1d2-250379444f6a","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:47:20.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6c6850f6-15f0-451f-b1d2-250379444f6a","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:47:20.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6c6850f6-15f0-451f-b1d2-250379444f6a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:47:20.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6c6850f6-15f0-451f-b1d2-250379444f6a","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:47:20.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6c6850f6-15f0-451f-b1d2-250379444f6a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:47:20.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6c6850f6-15f0-451f-b1d2-250379444f6a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:47:20.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6c6850f6-15f0-451f-b1d2-250379444f6a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:47:20.304Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6c6850f6-15f0-451f-b1d2-250379444f6a","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:47:20.304Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6c6850f6-15f0-451f-b1d2-250379444f6a","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:47:20.304Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6c6850f6-15f0-451f-b1d2-250379444f6a","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"newHead":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"}}
{"level":"info","ts":"2026-07-13T15:47:20.304Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6c6850f6-15f0-451f-b1d2-250379444f6a","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:43:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:47:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:47:20.315Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"d52e1346-ce62-4c1a-a03e-3ebd39b0c003","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:47:20.315Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"2c5b3a36-9759-4f8d-acd3-a5037a2af008","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:47:20.315Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"d52e1346-ce62-4c1a-a03e-3ebd39b0c003","pod":"raycluster-auth-head-czcms","ip":"10.133.0.17"}
{"level":"info","ts":"2026-07-13T15:47:20.331Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"2c5b3a36-9759-4f8d-acd3-a5037a2af008","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-13T15:47:20.341Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"d52e1346-ce62-4c1a-a03e-3ebd39b0c003","certificate":"ray-head-cert-raycluster-auth","dnsNames":["raycluster-auth-head-svc","localhost","raycluster-auth-head-svc.test-ns-k9mkf.svc","raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local"],"ipAddresses":["10.133.0.17","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:47:20.344Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"2c5b3a36-9759-4f8d-acd3-a5037a2af008","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-13T15:47:20.344Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"2c5b3a36-9759-4f8d-acd3-a5037a2af008","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:47:20.344Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5ea0d3fa-7d1f-4bfe-b940-d081bc82d7ae"}
{"level":"info","ts":"2026-07-13T15:47:20.344Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5ea0d3fa-7d1f-4bfe-b940-d081bc82d7ae"}
{"level":"info","ts":"2026-07-13T15:47:20.344Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5ea0d3fa-7d1f-4bfe-b940-d081bc82d7ae","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:47:20.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5ea0d3fa-7d1f-4bfe-b940-d081bc82d7ae","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:47:20.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5ea0d3fa-7d1f-4bfe-b940-d081bc82d7ae","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:47:20.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5ea0d3fa-7d1f-4bfe-b940-d081bc82d7ae","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:47:20.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5ea0d3fa-7d1f-4bfe-b940-d081bc82d7ae","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:47:20.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5ea0d3fa-7d1f-4bfe-b940-d081bc82d7ae","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:47:20.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5ea0d3fa-7d1f-4bfe-b940-d081bc82d7ae","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:47:20.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5ea0d3fa-7d1f-4bfe-b940-d081bc82d7ae","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:47:20.345Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5ea0d3fa-7d1f-4bfe-b940-d081bc82d7ae","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:47:20.345Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5ea0d3fa-7d1f-4bfe-b940-d081bc82d7ae","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:47:20.345Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5ea0d3fa-7d1f-4bfe-b940-d081bc82d7ae","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:47:20.345Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5ea0d3fa-7d1f-4bfe-b940-d081bc82d7ae","seconds":300}
{"level":"info","ts":"2026-07-13T15:47:20.354Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"d52e1346-ce62-4c1a-a03e-3ebd39b0c003","certificate":"ray-worker-cert-raycluster-auth","dnsNames":["raycluster-auth-worker-svc","localhost","raycluster-auth-small-group","raycluster-auth-worker-svc.test-ns-k9mkf.svc","raycluster-auth-worker-svc.test-ns-k9mkf.svc.cluster.local","*.raycluster-auth-worker-svc.test-ns-k9mkf.svc","*.raycluster-auth-worker-svc.test-ns-k9mkf.svc.cluster.local","*-worker-*.test-ns-k9mkf.svc","*-worker-*.test-ns-k9mkf.svc.cluster.local"],"ipAddresses":["10.133.0.17","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:47:20.354Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"d52e1346-ce62-4c1a-a03e-3ebd39b0c003","rayCluster":"raycluster-auth","podIPs":["10.133.0.17"]}
{"level":"info","ts":"2026-07-13T15:47:22.315Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3a636067-38b1-400a-a1de-b5c3a0b898ee"}
{"level":"info","ts":"2026-07-13T15:47:22.316Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3a636067-38b1-400a-a1de-b5c3a0b898ee"}
{"level":"info","ts":"2026-07-13T15:47:22.316Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3a636067-38b1-400a-a1de-b5c3a0b898ee","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:47:22.316Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3a636067-38b1-400a-a1de-b5c3a0b898ee","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:47:22.316Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3a636067-38b1-400a-a1de-b5c3a0b898ee","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:47:22.316Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3a636067-38b1-400a-a1de-b5c3a0b898ee","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:47:22.316Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3a636067-38b1-400a-a1de-b5c3a0b898ee","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:47:22.316Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3a636067-38b1-400a-a1de-b5c3a0b898ee","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:47:22.316Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3a636067-38b1-400a-a1de-b5c3a0b898ee","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:47:22.316Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3a636067-38b1-400a-a1de-b5c3a0b898ee","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:47:22.316Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3a636067-38b1-400a-a1de-b5c3a0b898ee","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:47:22.316Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3a636067-38b1-400a-a1de-b5c3a0b898ee","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:47:22.316Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3a636067-38b1-400a-a1de-b5c3a0b898ee","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:47:22.316Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"3a636067-38b1-400a-a1de-b5c3a0b898ee","seconds":300}
{"level":"info","ts":"2026-07-13T15:47:25.316Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"02f19fe7-119e-4e35-8cd3-cfcc0c1ae96c"}
{"level":"info","ts":"2026-07-13T15:47:25.316Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"02f19fe7-119e-4e35-8cd3-cfcc0c1ae96c"}
{"level":"info","ts":"2026-07-13T15:47:25.316Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"02f19fe7-119e-4e35-8cd3-cfcc0c1ae96c","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:47:25.316Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"02f19fe7-119e-4e35-8cd3-cfcc0c1ae96c","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:47:25.316Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"02f19fe7-119e-4e35-8cd3-cfcc0c1ae96c","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:47:25.316Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"02f19fe7-119e-4e35-8cd3-cfcc0c1ae96c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:47:25.316Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"02f19fe7-119e-4e35-8cd3-cfcc0c1ae96c","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:47:25.316Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"02f19fe7-119e-4e35-8cd3-cfcc0c1ae96c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:47:25.316Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"02f19fe7-119e-4e35-8cd3-cfcc0c1ae96c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:47:25.316Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"02f19fe7-119e-4e35-8cd3-cfcc0c1ae96c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:47:25.316Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"02f19fe7-119e-4e35-8cd3-cfcc0c1ae96c","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:47:25.316Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"02f19fe7-119e-4e35-8cd3-cfcc0c1ae96c","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:47:25.316Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"02f19fe7-119e-4e35-8cd3-cfcc0c1ae96c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:47:25.316Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"02f19fe7-119e-4e35-8cd3-cfcc0c1ae96c","seconds":300}
{"level":"info","ts":"2026-07-13T15:47:28.331Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0e5f19e9-20cb-4111-af49-d58be67f367d"}
{"level":"info","ts":"2026-07-13T15:47:28.331Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0e5f19e9-20cb-4111-af49-d58be67f367d"}
{"level":"info","ts":"2026-07-13T15:47:28.331Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0e5f19e9-20cb-4111-af49-d58be67f367d","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:47:28.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0e5f19e9-20cb-4111-af49-d58be67f367d","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:47:28.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0e5f19e9-20cb-4111-af49-d58be67f367d","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:47:28.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0e5f19e9-20cb-4111-af49-d58be67f367d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:47:28.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0e5f19e9-20cb-4111-af49-d58be67f367d","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:47:28.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0e5f19e9-20cb-4111-af49-d58be67f367d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:47:28.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0e5f19e9-20cb-4111-af49-d58be67f367d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:47:28.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0e5f19e9-20cb-4111-af49-d58be67f367d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:47:28.331Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0e5f19e9-20cb-4111-af49-d58be67f367d","podName":"raycluster-auth-head-czcms","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:43:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:47:28.332Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0e5f19e9-20cb-4111-af49-d58be67f367d","podName":"raycluster-auth-head-czcms","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:43:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:47:28.332Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0e5f19e9-20cb-4111-af49-d58be67f367d","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:47:28.332Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0e5f19e9-20cb-4111-af49-d58be67f367d","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:47:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:47:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:47:28.344Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"fa016095-3a7b-42a1-b190-9a00e5c5c9e3","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:47:28.344Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"f939e427-8b84-448a-9313-5ed92706630b","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:47:28.344Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"f939e427-8b84-448a-9313-5ed92706630b","pod":"raycluster-auth-head-czcms","ip":"10.133.0.17"}
{"level":"info","ts":"2026-07-13T15:47:28.344Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"f939e427-8b84-448a-9313-5ed92706630b","pod":"raycluster-auth-small-group-worker-z4nbk","ip":"10.134.0.24"}
{"level":"info","ts":"2026-07-13T15:47:28.357Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"fa016095-3a7b-42a1-b190-9a00e5c5c9e3","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-13T15:47:28.357Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"f939e427-8b84-448a-9313-5ed92706630b","certificate":"ray-head-cert-raycluster-auth","dnsNames":["raycluster-auth-head-svc","localhost","raycluster-auth-head-svc.test-ns-k9mkf.svc","raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local"],"ipAddresses":["10.133.0.17","10.134.0.24","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:47:28.368Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"f939e427-8b84-448a-9313-5ed92706630b","certificate":"ray-worker-cert-raycluster-auth","dnsNames":["raycluster-auth-worker-svc","localhost","raycluster-auth-small-group","raycluster-auth-worker-svc.test-ns-k9mkf.svc","raycluster-auth-worker-svc.test-ns-k9mkf.svc.cluster.local","*.raycluster-auth-worker-svc.test-ns-k9mkf.svc","*.raycluster-auth-worker-svc.test-ns-k9mkf.svc.cluster.local","*-worker-*.test-ns-k9mkf.svc","*-worker-*.test-ns-k9mkf.svc.cluster.local"],"ipAddresses":["10.133.0.17","10.134.0.24","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:47:28.368Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"f939e427-8b84-448a-9313-5ed92706630b","rayCluster":"raycluster-auth","podIPs":["10.133.0.17","10.134.0.24"]}
{"level":"info","ts":"2026-07-13T15:47:28.371Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"fa016095-3a7b-42a1-b190-9a00e5c5c9e3","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-13T15:47:28.371Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"fa016095-3a7b-42a1-b190-9a00e5c5c9e3","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:47:30.344Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c8dd587b-fc9a-4f42-ac59-3b886bdd9c2f"}
{"level":"info","ts":"2026-07-13T15:47:30.345Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c8dd587b-fc9a-4f42-ac59-3b886bdd9c2f"}
{"level":"info","ts":"2026-07-13T15:47:30.345Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c8dd587b-fc9a-4f42-ac59-3b886bdd9c2f","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:47:30.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c8dd587b-fc9a-4f42-ac59-3b886bdd9c2f","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:47:30.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c8dd587b-fc9a-4f42-ac59-3b886bdd9c2f","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:47:30.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c8dd587b-fc9a-4f42-ac59-3b886bdd9c2f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:47:30.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c8dd587b-fc9a-4f42-ac59-3b886bdd9c2f","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:47:30.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c8dd587b-fc9a-4f42-ac59-3b886bdd9c2f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:47:30.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c8dd587b-fc9a-4f42-ac59-3b886bdd9c2f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:47:30.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c8dd587b-fc9a-4f42-ac59-3b886bdd9c2f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:47:30.345Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c8dd587b-fc9a-4f42-ac59-3b886bdd9c2f","podName":"raycluster-auth-head-czcms","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:43:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:47:30.345Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c8dd587b-fc9a-4f42-ac59-3b886bdd9c2f","podName":"raycluster-auth-head-czcms","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:43:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:47:30.345Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c8dd587b-fc9a-4f42-ac59-3b886bdd9c2f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:47:30.345Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c8dd587b-fc9a-4f42-ac59-3b886bdd9c2f","seconds":300}
{"level":"info","ts":"2026-07-13T15:47:39.363Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"62087c4e-1fcf-453e-8165-c3eedccd0c3b"}
{"level":"info","ts":"2026-07-13T15:47:39.363Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"62087c4e-1fcf-453e-8165-c3eedccd0c3b"}
{"level":"info","ts":"2026-07-13T15:47:39.363Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"62087c4e-1fcf-453e-8165-c3eedccd0c3b","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:47:39.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"62087c4e-1fcf-453e-8165-c3eedccd0c3b","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:47:39.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"62087c4e-1fcf-453e-8165-c3eedccd0c3b","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:47:39.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"62087c4e-1fcf-453e-8165-c3eedccd0c3b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:47:39.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"62087c4e-1fcf-453e-8165-c3eedccd0c3b","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:47:39.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"62087c4e-1fcf-453e-8165-c3eedccd0c3b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:47:39.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"62087c4e-1fcf-453e-8165-c3eedccd0c3b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:47:39.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"62087c4e-1fcf-453e-8165-c3eedccd0c3b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:47:39.363Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"62087c4e-1fcf-453e-8165-c3eedccd0c3b","podName":"raycluster-auth-small-group-worker-z4nbk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:47:39.363Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"62087c4e-1fcf-453e-8165-c3eedccd0c3b","podName":"raycluster-auth-small-group-worker-z4nbk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:47:39.363Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"62087c4e-1fcf-453e-8165-c3eedccd0c3b","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:47:39Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:47:39.363Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"62087c4e-1fcf-453e-8165-c3eedccd0c3b","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:47:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:47:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:47:39Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:47:39.375Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"7de87db2-90fe-4fc8-a75b-b74fb6851321","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:47:39.375Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"3f4908e2-47c6-410d-9dd2-0a9c2f0aa164","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:47:39.375Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"7de87db2-90fe-4fc8-a75b-b74fb6851321","pod":"raycluster-auth-head-czcms","ip":"10.133.0.17"}
{"level":"info","ts":"2026-07-13T15:47:39.375Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"7de87db2-90fe-4fc8-a75b-b74fb6851321","pod":"raycluster-auth-small-group-worker-z4nbk","ip":"10.134.0.24"}
{"level":"info","ts":"2026-07-13T15:47:39.375Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"7de87db2-90fe-4fc8-a75b-b74fb6851321","rayCluster":"raycluster-auth","podIPs":["10.133.0.17","10.134.0.24"]}
{"level":"info","ts":"2026-07-13T15:47:39.389Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"3f4908e2-47c6-410d-9dd2-0a9c2f0aa164","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-13T15:47:39.398Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"3f4908e2-47c6-410d-9dd2-0a9c2f0aa164","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-13T15:47:39.398Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"3f4908e2-47c6-410d-9dd2-0a9c2f0aa164","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:47:41.375Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"04c63553-4ada-485b-b4da-f639387fe863"}
{"level":"info","ts":"2026-07-13T15:47:41.375Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"04c63553-4ada-485b-b4da-f639387fe863"}
{"level":"info","ts":"2026-07-13T15:47:41.375Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"04c63553-4ada-485b-b4da-f639387fe863","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:47:41.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"04c63553-4ada-485b-b4da-f639387fe863","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:47:41.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"04c63553-4ada-485b-b4da-f639387fe863","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:47:41.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"04c63553-4ada-485b-b4da-f639387fe863","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:47:41.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"04c63553-4ada-485b-b4da-f639387fe863","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:47:41.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"04c63553-4ada-485b-b4da-f639387fe863","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:47:41.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"04c63553-4ada-485b-b4da-f639387fe863","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:47:41.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"04c63553-4ada-485b-b4da-f639387fe863","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:47:41.376Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"04c63553-4ada-485b-b4da-f639387fe863","podName":"raycluster-auth-small-group-worker-z4nbk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:47:41.376Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"04c63553-4ada-485b-b4da-f639387fe863","podName":"raycluster-auth-small-group-worker-z4nbk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:47:41.376Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"04c63553-4ada-485b-b4da-f639387fe863","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:47:41.376Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"04c63553-4ada-485b-b4da-f639387fe863","seconds":300}
{"level":"info","ts":"2026-07-13T15:47:55.132Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"33cabc12-05f1-4ea8-b220-0e5481e7d9df","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:47:55.132Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"33cabc12-05f1-4ea8-b220-0e5481e7d9df","pod":"raycluster-auth-small-group-worker-z4nbk","ip":"10.134.0.24"}
{"level":"info","ts":"2026-07-13T15:47:55.132Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"33cabc12-05f1-4ea8-b220-0e5481e7d9df","pod":"raycluster-auth-head-czcms","ip":"10.133.0.17"}
{"level":"info","ts":"2026-07-13T15:47:55.132Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"33cabc12-05f1-4ea8-b220-0e5481e7d9df","rayCluster":"raycluster-auth","podIPs":["10.134.0.24","10.133.0.17"]}
{"level":"info","ts":"2026-07-13T15:48:40.656Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5c185f97-71e1-4df8-9ffc-80bc8cf5910a"}
{"level":"info","ts":"2026-07-13T15:48:40.656Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5c185f97-71e1-4df8-9ffc-80bc8cf5910a"}
{"level":"info","ts":"2026-07-13T15:48:40.656Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5c185f97-71e1-4df8-9ffc-80bc8cf5910a","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:48:40.656Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5c185f97-71e1-4df8-9ffc-80bc8cf5910a","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:48:40.656Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5c185f97-71e1-4df8-9ffc-80bc8cf5910a","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:48:40.656Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5c185f97-71e1-4df8-9ffc-80bc8cf5910a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:48:40.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5c185f97-71e1-4df8-9ffc-80bc8cf5910a","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:48:40.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5c185f97-71e1-4df8-9ffc-80bc8cf5910a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:48:40.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5c185f97-71e1-4df8-9ffc-80bc8cf5910a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:48:40.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5c185f97-71e1-4df8-9ffc-80bc8cf5910a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:48:40.657Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5c185f97-71e1-4df8-9ffc-80bc8cf5910a","podName":"raycluster-auth-small-group-worker-z4nbk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:48:40.657Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5c185f97-71e1-4df8-9ffc-80bc8cf5910a","podName":"raycluster-auth-small-group-worker-z4nbk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:48:40.657Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5c185f97-71e1-4df8-9ffc-80bc8cf5910a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:48:40.657Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5c185f97-71e1-4df8-9ffc-80bc8cf5910a","seconds":300}
{"level":"info","ts":"2026-07-13T15:48:41.664Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f02242e2-5b20-4888-9b95-7fad37bd02ad"}
{"level":"info","ts":"2026-07-13T15:48:41.664Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f02242e2-5b20-4888-9b95-7fad37bd02ad"}
{"level":"info","ts":"2026-07-13T15:48:41.664Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f02242e2-5b20-4888-9b95-7fad37bd02ad","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:48:41.664Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f02242e2-5b20-4888-9b95-7fad37bd02ad","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:48:41.664Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f02242e2-5b20-4888-9b95-7fad37bd02ad","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:48:41.664Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f02242e2-5b20-4888-9b95-7fad37bd02ad","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:48:41.664Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f02242e2-5b20-4888-9b95-7fad37bd02ad","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:48:41.664Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f02242e2-5b20-4888-9b95-7fad37bd02ad","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:48:41.664Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f02242e2-5b20-4888-9b95-7fad37bd02ad","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:48:41.664Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f02242e2-5b20-4888-9b95-7fad37bd02ad","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:48:41.664Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f02242e2-5b20-4888-9b95-7fad37bd02ad","podName":"raycluster-auth-small-group-worker-z4nbk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:48:41.665Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f02242e2-5b20-4888-9b95-7fad37bd02ad","podName":"raycluster-auth-small-group-worker-z4nbk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:48:41.665Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f02242e2-5b20-4888-9b95-7fad37bd02ad","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:48:41.665Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f02242e2-5b20-4888-9b95-7fad37bd02ad","seconds":300}
{"level":"info","ts":"2026-07-13T15:48:51.707Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5fee18ad-a19a-400f-b77d-fe73cdd31dae"}
{"level":"info","ts":"2026-07-13T15:48:51.707Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5fee18ad-a19a-400f-b77d-fe73cdd31dae"}
{"level":"info","ts":"2026-07-13T15:48:51.707Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5fee18ad-a19a-400f-b77d-fe73cdd31dae","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:48:51.707Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5fee18ad-a19a-400f-b77d-fe73cdd31dae","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:48:51.707Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5fee18ad-a19a-400f-b77d-fe73cdd31dae","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:48:51.707Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5fee18ad-a19a-400f-b77d-fe73cdd31dae","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:48:51.707Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5fee18ad-a19a-400f-b77d-fe73cdd31dae","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:48:51.707Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5fee18ad-a19a-400f-b77d-fe73cdd31dae","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:48:51.707Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5fee18ad-a19a-400f-b77d-fe73cdd31dae","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:48:51.707Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5fee18ad-a19a-400f-b77d-fe73cdd31dae","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:48:51.707Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5fee18ad-a19a-400f-b77d-fe73cdd31dae","podName":"raycluster-auth-small-group-worker-z4nbk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:48:51.707Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5fee18ad-a19a-400f-b77d-fe73cdd31dae","podName":"raycluster-auth-small-group-worker-z4nbk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:48:51.707Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5fee18ad-a19a-400f-b77d-fe73cdd31dae","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:48:51.707Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"5fee18ad-a19a-400f-b77d-fe73cdd31dae","seconds":300}
{"level":"info","ts":"2026-07-13T15:48:52.712Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"843b20de-f274-448d-b7eb-0379cce32452"}
{"level":"info","ts":"2026-07-13T15:48:52.712Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"843b20de-f274-448d-b7eb-0379cce32452"}
{"level":"info","ts":"2026-07-13T15:48:52.712Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"843b20de-f274-448d-b7eb-0379cce32452","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:48:52.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"843b20de-f274-448d-b7eb-0379cce32452","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:48:52.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"843b20de-f274-448d-b7eb-0379cce32452","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:48:52.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"843b20de-f274-448d-b7eb-0379cce32452","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:48:52.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"843b20de-f274-448d-b7eb-0379cce32452","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:48:52.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"843b20de-f274-448d-b7eb-0379cce32452","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:48:52.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"843b20de-f274-448d-b7eb-0379cce32452","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:48:52.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"843b20de-f274-448d-b7eb-0379cce32452","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:48:52.712Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"843b20de-f274-448d-b7eb-0379cce32452","podName":"raycluster-auth-small-group-worker-z4nbk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:43:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:48:52.712Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"843b20de-f274-448d-b7eb-0379cce32452","podName":"raycluster-auth-small-group-worker-z4nbk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:43:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:48:52.712Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"843b20de-f274-448d-b7eb-0379cce32452","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:48:52.712Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"843b20de-f274-448d-b7eb-0379cce32452","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:47:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:47:39Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:48:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:47:39Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:48:52.724Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"794fb03a-137f-4088-b509-9abd80429963","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:48:52.724Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"4756a922-eb90-4640-96e8-d8735436fceb","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:48:52.724Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"794fb03a-137f-4088-b509-9abd80429963","pod":"raycluster-auth-head-czcms","ip":"10.133.0.17"}
{"level":"info","ts":"2026-07-13T15:48:52.724Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"794fb03a-137f-4088-b509-9abd80429963","pod":"raycluster-auth-small-group-worker-z4nbk","ip":"10.134.0.24"}
{"level":"info","ts":"2026-07-13T15:48:52.725Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"794fb03a-137f-4088-b509-9abd80429963","rayCluster":"raycluster-auth","podIPs":["10.133.0.17","10.134.0.24"]}
{"level":"info","ts":"2026-07-13T15:48:52.740Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"4756a922-eb90-4640-96e8-d8735436fceb","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-13T15:48:52.751Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"4756a922-eb90-4640-96e8-d8735436fceb","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-13T15:48:52.751Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"4756a922-eb90-4640-96e8-d8735436fceb","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:48:54.725Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4d03f186-3c58-488a-9a7f-1a788dfa84ac"}
{"level":"info","ts":"2026-07-13T15:48:54.725Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4d03f186-3c58-488a-9a7f-1a788dfa84ac"}
{"level":"info","ts":"2026-07-13T15:48:54.725Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4d03f186-3c58-488a-9a7f-1a788dfa84ac","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:48:54.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4d03f186-3c58-488a-9a7f-1a788dfa84ac","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:48:54.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4d03f186-3c58-488a-9a7f-1a788dfa84ac","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:48:54.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4d03f186-3c58-488a-9a7f-1a788dfa84ac","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:48:54.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4d03f186-3c58-488a-9a7f-1a788dfa84ac","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:48:54.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4d03f186-3c58-488a-9a7f-1a788dfa84ac","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:48:54.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4d03f186-3c58-488a-9a7f-1a788dfa84ac","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:48:54.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4d03f186-3c58-488a-9a7f-1a788dfa84ac","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:48:54.725Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4d03f186-3c58-488a-9a7f-1a788dfa84ac","podName":"raycluster-auth-small-group-worker-z4nbk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:43:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:48:54.725Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4d03f186-3c58-488a-9a7f-1a788dfa84ac","podName":"raycluster-auth-small-group-worker-z4nbk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:43:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:48:54.725Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4d03f186-3c58-488a-9a7f-1a788dfa84ac","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:48:54.725Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"4d03f186-3c58-488a-9a7f-1a788dfa84ac","seconds":300}
{"level":"info","ts":"2026-07-13T15:48:55.133Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"f2360fe5-4760-447a-bae3-81e636c8059c","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:48:55.133Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"f2360fe5-4760-447a-bae3-81e636c8059c","pod":"raycluster-auth-head-czcms","ip":"10.133.0.17"}
{"level":"info","ts":"2026-07-13T15:48:55.133Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"f2360fe5-4760-447a-bae3-81e636c8059c","pod":"raycluster-auth-small-group-worker-z4nbk","ip":"10.134.0.24"}
{"level":"info","ts":"2026-07-13T15:48:55.133Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"f2360fe5-4760-447a-bae3-81e636c8059c","rayCluster":"raycluster-auth","podIPs":["10.133.0.17","10.134.0.24"]}
{"level":"info","ts":"2026-07-13T15:49:03.733Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d3b9840-702f-4f90-bead-2af197f12668"}
{"level":"info","ts":"2026-07-13T15:49:03.733Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d3b9840-702f-4f90-bead-2af197f12668"}
{"level":"info","ts":"2026-07-13T15:49:03.733Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d3b9840-702f-4f90-bead-2af197f12668","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:03.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d3b9840-702f-4f90-bead-2af197f12668","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:03.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d3b9840-702f-4f90-bead-2af197f12668","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:03.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d3b9840-702f-4f90-bead-2af197f12668","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:03.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d3b9840-702f-4f90-bead-2af197f12668","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:03.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d3b9840-702f-4f90-bead-2af197f12668","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:03.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d3b9840-702f-4f90-bead-2af197f12668","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:03.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d3b9840-702f-4f90-bead-2af197f12668","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:03.733Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d3b9840-702f-4f90-bead-2af197f12668","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-13T15:49:03.733Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d3b9840-702f-4f90-bead-2af197f12668","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:48:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:47:39Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:03Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:03Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:47:39Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:03Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:03.744Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"03e5eefc-37d2-4d97-9add-358741b4551b","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:03.744Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"57b77895-bda0-48df-b911-075f46ffca50","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:03.745Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"03e5eefc-37d2-4d97-9add-358741b4551b","pod":"raycluster-auth-small-group-worker-z4nbk","ip":"10.134.0.24"}
{"level":"info","ts":"2026-07-13T15:49:03.745Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"03e5eefc-37d2-4d97-9add-358741b4551b","pod":"raycluster-auth-head-czcms","ip":"10.133.0.17"}
{"level":"info","ts":"2026-07-13T15:49:03.745Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"03e5eefc-37d2-4d97-9add-358741b4551b","rayCluster":"raycluster-auth","podIPs":["10.134.0.24","10.133.0.17"]}
{"level":"info","ts":"2026-07-13T15:49:03.753Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"57b77895-bda0-48df-b911-075f46ffca50","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-07-13T15:49:03.763Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"57b77895-bda0-48df-b911-075f46ffca50","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-07-13T15:49:03.763Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"57b77895-bda0-48df-b911-075f46ffca50","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:04.538Z","logger":"raycluster-resource","msg":"default","name":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.538Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster","namespace":"test-ns-qbjjb"}
{"level":"info","ts":"2026-07-13T15:49:04.549Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a010acc8-94c2-433a-8c8e-fe7d177d45a3","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.549Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a010acc8-94c2-433a-8c8e-fe7d177d45a3","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.549Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"da87ea8f-c0bb-4ed4-888b-d6fc5bfc00c4","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.549Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"3e47a6d4-5e5d-4ba1-8d55-28024219d673"}
{"level":"info","ts":"2026-07-13T15:49:04.549Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"3e47a6d4-5e5d-4ba1-8d55-28024219d673"}
{"level":"info","ts":"2026-07-13T15:49:04.549Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"f1d73975-584a-435d-bd40-eb98d2c7a6f4","namespacedName":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T15:49:04.550Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"f1d73975-584a-435d-bd40-eb98d2c7a6f4","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.550Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"f1d73975-584a-435d-bd40-eb98d2c7a6f4","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.550Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"f1d73975-584a-435d-bd40-eb98d2c7a6f4","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.555Z","logger":"raycluster-resource","msg":"default","name":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.555Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster","namespace":"test-ns-qbjjb"}
{"level":"info","ts":"2026-07-13T15:49:04.556Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"3e47a6d4-5e5d-4ba1-8d55-28024219d673","name":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:04.556Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"3e47a6d4-5e5d-4ba1-8d55-28024219d673","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-07-13T15:49:04.556Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"3e47a6d4-5e5d-4ba1-8d55-28024219d673","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:04.556Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"3e47a6d4-5e5d-4ba1-8d55-28024219d673","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-07-13T15:49:04.556Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"3e47a6d4-5e5d-4ba1-8d55-28024219d673","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:04.556Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"3e47a6d4-5e5d-4ba1-8d55-28024219d673","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:04.556Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"3e47a6d4-5e5d-4ba1-8d55-28024219d673","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:04.556Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"3e47a6d4-5e5d-4ba1-8d55-28024219d673","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:49:04.556Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"3e47a6d4-5e5d-4ba1-8d55-28024219d673","name":"raycluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:04.621Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"da87ea8f-c0bb-4ed4-888b-d6fc5bfc00c4","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.621Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"11f28409-6360-4ee2-a78c-438f4b208a97","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.621Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"3e47a6d4-5e5d-4ba1-8d55-28024219d673","name":"raycluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-qbjjb","name":"raycluster"}}
{"level":"info","ts":"2026-07-13T15:49:04.621Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"3e47a6d4-5e5d-4ba1-8d55-28024219d673"}
{"level":"error","ts":"2026-07-13T15:49:04.621Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"3e47a6d4-5e5d-4ba1-8d55-28024219d673","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:04.621Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a07480bf-e793-4a31-8bfb-35e76c128a7c"}
{"level":"info","ts":"2026-07-13T15:49:04.621Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a07480bf-e793-4a31-8bfb-35e76c128a7c"}
{"level":"info","ts":"2026-07-13T15:49:04.621Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a010acc8-94c2-433a-8c8e-fe7d177d45a3","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.621Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a010acc8-94c2-433a-8c8e-fe7d177d45a3","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.621Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a07480bf-e793-4a31-8bfb-35e76c128a7c","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:04.622Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a07480bf-e793-4a31-8bfb-35e76c128a7c","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-07-13T15:49:04.622Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a07480bf-e793-4a31-8bfb-35e76c128a7c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:04.622Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a07480bf-e793-4a31-8bfb-35e76c128a7c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-07-13T15:49:04.622Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a07480bf-e793-4a31-8bfb-35e76c128a7c","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:04.622Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a07480bf-e793-4a31-8bfb-35e76c128a7c","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:04.622Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a07480bf-e793-4a31-8bfb-35e76c128a7c","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:04.622Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a07480bf-e793-4a31-8bfb-35e76c128a7c","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:49:04.622Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a07480bf-e793-4a31-8bfb-35e76c128a7c","name":"raycluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:04.627Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"f1d73975-584a-435d-bd40-eb98d2c7a6f4","name":"raycluster-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:04.632Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a07480bf-e793-4a31-8bfb-35e76c128a7c"}
{"level":"error","ts":"2026-07-13T15:49:04.632Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a07480bf-e793-4a31-8bfb-35e76c128a7c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:04.632Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e84b6cc5-f0aa-4e1f-834e-54ebcbd02a6e"}
{"level":"info","ts":"2026-07-13T15:49:04.632Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e84b6cc5-f0aa-4e1f-834e-54ebcbd02a6e"}
{"level":"info","ts":"2026-07-13T15:49:04.632Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e84b6cc5-f0aa-4e1f-834e-54ebcbd02a6e","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:04.632Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e84b6cc5-f0aa-4e1f-834e-54ebcbd02a6e","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-07-13T15:49:04.632Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e84b6cc5-f0aa-4e1f-834e-54ebcbd02a6e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:04.632Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e84b6cc5-f0aa-4e1f-834e-54ebcbd02a6e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-07-13T15:49:04.632Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e84b6cc5-f0aa-4e1f-834e-54ebcbd02a6e","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:04.632Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e84b6cc5-f0aa-4e1f-834e-54ebcbd02a6e","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:04.632Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e84b6cc5-f0aa-4e1f-834e-54ebcbd02a6e","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:04.632Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e84b6cc5-f0aa-4e1f-834e-54ebcbd02a6e","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:49:04.632Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e84b6cc5-f0aa-4e1f-834e-54ebcbd02a6e","name":"raycluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:04.634Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a010acc8-94c2-433a-8c8e-fe7d177d45a3","rayCluster":"raycluster","secret":"ca-secret"}
{"level":"info","ts":"2026-07-13T15:49:04.634Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a010acc8-94c2-433a-8c8e-fe7d177d45a3","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.634Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"f1d73975-584a-435d-bd40-eb98d2c7a6f4","name":"kuberay-gateway-access","namespace":"test-ns-qbjjb","operation":"created"}
{"level":"info","ts":"2026-07-13T15:49:04.635Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"11f28409-6360-4ee2-a78c-438f4b208a97","name":"raycluster-head"}
{"level":"info","ts":"2026-07-13T15:49:04.638Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e84b6cc5-f0aa-4e1f-834e-54ebcbd02a6e","name":"raycluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-qbjjb","name":"raycluster"}}
{"level":"info","ts":"2026-07-13T15:49:04.638Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e84b6cc5-f0aa-4e1f-834e-54ebcbd02a6e"}
{"level":"error","ts":"2026-07-13T15:49:04.638Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e84b6cc5-f0aa-4e1f-834e-54ebcbd02a6e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:04.642Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"89c11326-ab90-424e-9de2-faec5e358e82"}
{"level":"info","ts":"2026-07-13T15:49:04.642Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"89c11326-ab90-424e-9de2-faec5e358e82"}
{"level":"info","ts":"2026-07-13T15:49:04.642Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"89c11326-ab90-424e-9de2-faec5e358e82","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:04.642Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"89c11326-ab90-424e-9de2-faec5e358e82","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-07-13T15:49:04.642Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"89c11326-ab90-424e-9de2-faec5e358e82","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:04.642Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"89c11326-ab90-424e-9de2-faec5e358e82","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-07-13T15:49:04.642Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"89c11326-ab90-424e-9de2-faec5e358e82","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:04.642Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"89c11326-ab90-424e-9de2-faec5e358e82","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:04.642Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"89c11326-ab90-424e-9de2-faec5e358e82","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:04.642Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"89c11326-ab90-424e-9de2-faec5e358e82"}
{"level":"error","ts":"2026-07-13T15:49:04.642Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"89c11326-ab90-424e-9de2-faec5e358e82","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:04.644Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"f1d73975-584a-435d-bd40-eb98d2c7a6f4","name":"test-ns-qbjjb-raycluster","operation":"created"}
{"level":"info","ts":"2026-07-13T15:49:04.645Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a010acc8-94c2-433a-8c8e-fe7d177d45a3","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.651Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"11f28409-6360-4ee2-a78c-438f4b208a97","name":"raycluster-workers"}
{"level":"info","ts":"2026-07-13T15:49:04.651Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"11f28409-6360-4ee2-a78c-438f4b208a97","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.651Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8395eae4-ced3-4d18-bdf9-11d98c9b5a5c","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.669Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"f1d73975-584a-435d-bd40-eb98d2c7a6f4","name":"kube-rbac-proxy-config-raycluster","operation":"created"}
{"level":"error","ts":"2026-07-13T15:49:04.680Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"f1d73975-584a-435d-bd40-eb98d2c7a6f4","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:04.680Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"f1d73975-584a-435d-bd40-eb98d2c7a6f4"}
{"level":"error","ts":"2026-07-13T15:49:04.680Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"f1d73975-584a-435d-bd40-eb98d2c7a6f4","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:04.680Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"fd53ad31-06d2-491a-814d-558d0a20df11","namespacedName":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T15:49:04.680Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"fd53ad31-06d2-491a-814d-558d0a20df11","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.680Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"fd53ad31-06d2-491a-814d-558d0a20df11","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.721Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c434a54b-e284-48e0-b3fe-a7a6e2eee18c"}
{"level":"info","ts":"2026-07-13T15:49:04.721Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c434a54b-e284-48e0-b3fe-a7a6e2eee18c"}
{"level":"info","ts":"2026-07-13T15:49:04.721Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c434a54b-e284-48e0-b3fe-a7a6e2eee18c","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:04.721Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c434a54b-e284-48e0-b3fe-a7a6e2eee18c","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-07-13T15:49:04.721Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c434a54b-e284-48e0-b3fe-a7a6e2eee18c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:04.721Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c434a54b-e284-48e0-b3fe-a7a6e2eee18c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-07-13T15:49:04.721Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c434a54b-e284-48e0-b3fe-a7a6e2eee18c","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:04.721Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c434a54b-e284-48e0-b3fe-a7a6e2eee18c","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:04.721Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c434a54b-e284-48e0-b3fe-a7a6e2eee18c","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:04.721Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c434a54b-e284-48e0-b3fe-a7a6e2eee18c"}
{"level":"error","ts":"2026-07-13T15:49:04.721Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c434a54b-e284-48e0-b3fe-a7a6e2eee18c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:04.721Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a010acc8-94c2-433a-8c8e-fe7d177d45a3","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:49:04.821Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f353778e-7f35-4a9e-8840-27637e6e0fc1"}
{"level":"info","ts":"2026-07-13T15:49:04.821Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f353778e-7f35-4a9e-8840-27637e6e0fc1"}
{"level":"info","ts":"2026-07-13T15:49:04.821Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f353778e-7f35-4a9e-8840-27637e6e0fc1","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:04.821Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f353778e-7f35-4a9e-8840-27637e6e0fc1","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-07-13T15:49:04.821Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f353778e-7f35-4a9e-8840-27637e6e0fc1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:04.821Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f353778e-7f35-4a9e-8840-27637e6e0fc1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-07-13T15:49:04.821Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f353778e-7f35-4a9e-8840-27637e6e0fc1","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:04.821Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f353778e-7f35-4a9e-8840-27637e6e0fc1","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:04.821Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f353778e-7f35-4a9e-8840-27637e6e0fc1","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:04.821Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f353778e-7f35-4a9e-8840-27637e6e0fc1"}
{"level":"error","ts":"2026-07-13T15:49:04.821Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f353778e-7f35-4a9e-8840-27637e6e0fc1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:04.822Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a010acc8-94c2-433a-8c8e-fe7d177d45a3","rayCluster":"raycluster","podIPs":[]}
{"level":"error","ts":"2026-07-13T15:49:04.822Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a010acc8-94c2-433a-8c8e-fe7d177d45a3","error":"Certificate.cert-manager.io \"ray-worker-cert-raycluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:04.822Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a010acc8-94c2-433a-8c8e-fe7d177d45a3"}
{"level":"error","ts":"2026-07-13T15:49:04.822Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a010acc8-94c2-433a-8c8e-fe7d177d45a3","error":"Certificate.cert-manager.io \"ray-worker-cert-raycluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:04.822Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"0375075b-e78d-4b55-976f-aa1390880101","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.822Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"fd53ad31-06d2-491a-814d-558d0a20df11","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.822Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"50b4f434-c6e3-4268-a39c-bc39787a2f6b","namespacedName":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T15:49:04.822Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"50b4f434-c6e3-4268-a39c-bc39787a2f6b","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.822Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"50b4f434-c6e3-4268-a39c-bc39787a2f6b","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.921Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8395eae4-ced3-4d18-bdf9-11d98c9b5a5c","name":"raycluster-head"}
{"level":"error","ts":"2026-07-13T15:49:04.921Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"0375075b-e78d-4b55-976f-aa1390880101","error":"certificates.cert-manager.io \"ray-worker-cert-raycluster\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:49:04.922Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"0375075b-e78d-4b55-976f-aa1390880101","error":"certificates.cert-manager.io \"ray-worker-cert-raycluster\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:04.922Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"0375075b-e78d-4b55-976f-aa1390880101"}
{"level":"error","ts":"2026-07-13T15:49:04.922Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"0375075b-e78d-4b55-976f-aa1390880101","error":"certificates.cert-manager.io \"ray-worker-cert-raycluster\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:04.922Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"018c3c30-db60-4c31-943c-353f3bc54bbd","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.922Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"018c3c30-db60-4c31-943c-353f3bc54bbd"}
{"level":"info","ts":"2026-07-13T15:49:04.922Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"50b4f434-c6e3-4268-a39c-bc39787a2f6b","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.932Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"fa1ac9f6-0c6b-417c-80f3-b3fe6adbfd69","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.932Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"fa1ac9f6-0c6b-417c-80f3-b3fe6adbfd69"}
{"level":"info","ts":"2026-07-13T15:49:04.935Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8395eae4-ced3-4d18-bdf9-11d98c9b5a5c","name":"raycluster-workers"}
{"level":"info","ts":"2026-07-13T15:49:04.935Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8395eae4-ced3-4d18-bdf9-11d98c9b5a5c","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.936Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"951b13de-ed30-4ddd-815e-e717d8650694","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.945Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"951b13de-ed30-4ddd-815e-e717d8650694","name":"raycluster-head"}
{"level":"info","ts":"2026-07-13T15:49:04.955Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"951b13de-ed30-4ddd-815e-e717d8650694","name":"raycluster-workers"}
{"level":"info","ts":"2026-07-13T15:49:04.955Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"951b13de-ed30-4ddd-815e-e717d8650694","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:04.982Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9afa78f0-d2b0-4403-a9a8-3b26d1989929"}
{"level":"info","ts":"2026-07-13T15:49:04.982Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9afa78f0-d2b0-4403-a9a8-3b26d1989929"}
{"level":"info","ts":"2026-07-13T15:49:04.982Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9afa78f0-d2b0-4403-a9a8-3b26d1989929","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:04.982Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9afa78f0-d2b0-4403-a9a8-3b26d1989929","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-07-13T15:49:04.982Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9afa78f0-d2b0-4403-a9a8-3b26d1989929","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:04.982Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9afa78f0-d2b0-4403-a9a8-3b26d1989929","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-07-13T15:49:04.982Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9afa78f0-d2b0-4403-a9a8-3b26d1989929","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:04.982Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9afa78f0-d2b0-4403-a9a8-3b26d1989929","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:04.982Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9afa78f0-d2b0-4403-a9a8-3b26d1989929","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:04.982Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9afa78f0-d2b0-4403-a9a8-3b26d1989929"}
{"level":"error","ts":"2026-07-13T15:49:04.982Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9afa78f0-d2b0-4403-a9a8-3b26d1989929","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:05.302Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a69059af-c594-4fe3-8e43-7cde6e59169c"}
{"level":"info","ts":"2026-07-13T15:49:05.302Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a69059af-c594-4fe3-8e43-7cde6e59169c"}
{"level":"info","ts":"2026-07-13T15:49:05.303Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a69059af-c594-4fe3-8e43-7cde6e59169c","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:05.303Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a69059af-c594-4fe3-8e43-7cde6e59169c","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-07-13T15:49:05.303Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a69059af-c594-4fe3-8e43-7cde6e59169c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:05.303Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a69059af-c594-4fe3-8e43-7cde6e59169c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-07-13T15:49:05.303Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a69059af-c594-4fe3-8e43-7cde6e59169c","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:05.303Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a69059af-c594-4fe3-8e43-7cde6e59169c","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:05.303Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a69059af-c594-4fe3-8e43-7cde6e59169c","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:05.303Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a69059af-c594-4fe3-8e43-7cde6e59169c"}
{"level":"error","ts":"2026-07-13T15:49:05.303Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a69059af-c594-4fe3-8e43-7cde6e59169c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:05.745Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"b721e01b-ced6-47ea-957a-0a2c1a6b64bb"}
{"level":"info","ts":"2026-07-13T15:49:05.745Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"b721e01b-ced6-47ea-957a-0a2c1a6b64bb"}
{"level":"info","ts":"2026-07-13T15:49:05.745Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"b721e01b-ced6-47ea-957a-0a2c1a6b64bb","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:05.745Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"b721e01b-ced6-47ea-957a-0a2c1a6b64bb","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:05.745Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"b721e01b-ced6-47ea-957a-0a2c1a6b64bb","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:05.745Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"b721e01b-ced6-47ea-957a-0a2c1a6b64bb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:05.746Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"b721e01b-ced6-47ea-957a-0a2c1a6b64bb","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:05.746Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"b721e01b-ced6-47ea-957a-0a2c1a6b64bb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:05.746Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"b721e01b-ced6-47ea-957a-0a2c1a6b64bb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:05.746Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"b721e01b-ced6-47ea-957a-0a2c1a6b64bb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:05.746Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"b721e01b-ced6-47ea-957a-0a2c1a6b64bb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:05.746Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"b721e01b-ced6-47ea-957a-0a2c1a6b64bb","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:05.944Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"051fd40b-6e00-47b6-98a2-4898166fc2f1"}
{"level":"info","ts":"2026-07-13T15:49:05.944Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"051fd40b-6e00-47b6-98a2-4898166fc2f1"}
{"level":"info","ts":"2026-07-13T15:49:05.944Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"051fd40b-6e00-47b6-98a2-4898166fc2f1","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:05.944Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"051fd40b-6e00-47b6-98a2-4898166fc2f1","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-07-13T15:49:05.944Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"051fd40b-6e00-47b6-98a2-4898166fc2f1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:05.944Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"051fd40b-6e00-47b6-98a2-4898166fc2f1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-07-13T15:49:05.944Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"051fd40b-6e00-47b6-98a2-4898166fc2f1","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:05.944Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"051fd40b-6e00-47b6-98a2-4898166fc2f1","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:05.944Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"051fd40b-6e00-47b6-98a2-4898166fc2f1","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:05.944Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"051fd40b-6e00-47b6-98a2-4898166fc2f1"}
{"level":"error","ts":"2026-07-13T15:49:05.944Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"051fd40b-6e00-47b6-98a2-4898166fc2f1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:07.225Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313"}
{"level":"info","ts":"2026-07-13T15:49:07.225Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313"}
{"level":"info","ts":"2026-07-13T15:49:07.225Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:07.225Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313"}
{"level":"info","ts":"2026-07-13T15:49:07.225Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:07.225Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313"}
{"level":"info","ts":"2026-07-13T15:49:07.225Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:07.225Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","cluster":"raycluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:07.225Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","cluster":"raycluster","authType":"IntegratedOAuth","serviceAccount":"raycluster-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T15:49:07.225Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster","ray.io/group":"headgroup","ray.io/identifier":"raycluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T15:49:07.225Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T15:49:07.225Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:07.225Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:07.225Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","enabled":true}
{"level":"info","ts":"2026-07-13T15:49:07.244Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","name":"raycluster-head-fdvcn"}
{"level":"info","ts":"2026-07-13T15:49:07.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:07.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:07.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T15:49:07.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:07.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T15:49:07.244Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313"}
{"level":"info","ts":"2026-07-13T15:49:07.244Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T15:49:07.244Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:49:07.244Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:49:07.244Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","enabled":true}
{"level":"info","ts":"2026-07-13T15:49:07.257Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","name":"raycluster-small-group-worker-96vng"}
{"level":"info","ts":"2026-07-13T15:49:07.257Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","podName":"raycluster-head-fdvcn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:07.257Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"raycluster-head-svc"},"newHead":{"podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"}}
{"level":"info","ts":"2026-07-13T15:49:07.257Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"50a672f5-880f-4b32-a9ad-07b1d38b1313","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:07.277Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"01d6ebd8-0bc3-4a3a-8d02-11d163d45e10","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:07.277Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"400e9355-8e90-4ea3-8697-778e4d275912","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:07.277Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"98a6d28c-2b84-4539-8ae3-18e777af5c5b"}
{"level":"info","ts":"2026-07-13T15:49:07.277Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"98a6d28c-2b84-4539-8ae3-18e777af5c5b"}
{"level":"info","ts":"2026-07-13T15:49:07.277Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"98a6d28c-2b84-4539-8ae3-18e777af5c5b","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:07.277Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"98a6d28c-2b84-4539-8ae3-18e777af5c5b","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:07.277Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"98a6d28c-2b84-4539-8ae3-18e777af5c5b","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:07.277Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"98a6d28c-2b84-4539-8ae3-18e777af5c5b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:07.277Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"98a6d28c-2b84-4539-8ae3-18e777af5c5b","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:07.277Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"98a6d28c-2b84-4539-8ae3-18e777af5c5b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:07.277Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"98a6d28c-2b84-4539-8ae3-18e777af5c5b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:07.277Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"98a6d28c-2b84-4539-8ae3-18e777af5c5b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:07.277Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"98a6d28c-2b84-4539-8ae3-18e777af5c5b","podName":"raycluster-head-fdvcn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:07.277Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"98a6d28c-2b84-4539-8ae3-18e777af5c5b","podName":"raycluster-head-fdvcn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:07.277Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"98a6d28c-2b84-4539-8ae3-18e777af5c5b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:07.277Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"98a6d28c-2b84-4539-8ae3-18e777af5c5b","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:07.321Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"01d6ebd8-0bc3-4a3a-8d02-11d163d45e10","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:49:07.321Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"400e9355-8e90-4ea3-8697-778e4d275912","name":"raycluster-head"}
{"level":"info","ts":"2026-07-13T15:49:07.321Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"87ff0f10-5542-4108-bcdb-00aea5cd4dff"}
{"level":"info","ts":"2026-07-13T15:49:07.321Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"87ff0f10-5542-4108-bcdb-00aea5cd4dff"}
{"level":"info","ts":"2026-07-13T15:49:07.321Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"87ff0f10-5542-4108-bcdb-00aea5cd4dff","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:07.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"87ff0f10-5542-4108-bcdb-00aea5cd4dff","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:07.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"87ff0f10-5542-4108-bcdb-00aea5cd4dff","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:07.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"87ff0f10-5542-4108-bcdb-00aea5cd4dff","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:07.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"87ff0f10-5542-4108-bcdb-00aea5cd4dff","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:07.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"87ff0f10-5542-4108-bcdb-00aea5cd4dff","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:07.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"87ff0f10-5542-4108-bcdb-00aea5cd4dff","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:07.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"87ff0f10-5542-4108-bcdb-00aea5cd4dff","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:07.322Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"87ff0f10-5542-4108-bcdb-00aea5cd4dff","podName":"raycluster-head-fdvcn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:07.322Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"87ff0f10-5542-4108-bcdb-00aea5cd4dff","podName":"raycluster-head-fdvcn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:07.322Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"87ff0f10-5542-4108-bcdb-00aea5cd4dff","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:49:07.322Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"87ff0f10-5542-4108-bcdb-00aea5cd4dff","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:07.332Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"400e9355-8e90-4ea3-8697-778e4d275912","name":"raycluster-workers"}
{"level":"info","ts":"2026-07-13T15:49:07.332Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"400e9355-8e90-4ea3-8697-778e4d275912","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:07.333Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37ec5d9e-1d2d-49f8-b7b8-62438f36379c"}
{"level":"info","ts":"2026-07-13T15:49:07.333Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37ec5d9e-1d2d-49f8-b7b8-62438f36379c"}
{"level":"info","ts":"2026-07-13T15:49:07.333Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37ec5d9e-1d2d-49f8-b7b8-62438f36379c","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:07.333Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"c9af65b8-885c-4fc9-b2c8-5d88efe32fa7","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:07.333Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"294dfc78-cc3f-486f-83d9-57df39578ffa","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:07.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37ec5d9e-1d2d-49f8-b7b8-62438f36379c","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:07.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37ec5d9e-1d2d-49f8-b7b8-62438f36379c","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:07.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37ec5d9e-1d2d-49f8-b7b8-62438f36379c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:07.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37ec5d9e-1d2d-49f8-b7b8-62438f36379c","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:07.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37ec5d9e-1d2d-49f8-b7b8-62438f36379c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:07.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37ec5d9e-1d2d-49f8-b7b8-62438f36379c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:07.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37ec5d9e-1d2d-49f8-b7b8-62438f36379c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:07.334Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"294dfc78-cc3f-486f-83d9-57df39578ffa","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:49:07.334Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37ec5d9e-1d2d-49f8-b7b8-62438f36379c","podName":"raycluster-head-fdvcn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:07.334Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37ec5d9e-1d2d-49f8-b7b8-62438f36379c","podName":"raycluster-head-fdvcn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:07.334Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37ec5d9e-1d2d-49f8-b7b8-62438f36379c","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:49:07.334Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37ec5d9e-1d2d-49f8-b7b8-62438f36379c","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:07.341Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37ec5d9e-1d2d-49f8-b7b8-62438f36379c","name":"raycluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-qbjjb","name":"raycluster"}}
{"level":"info","ts":"2026-07-13T15:49:07.341Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37ec5d9e-1d2d-49f8-b7b8-62438f36379c"}
{"level":"error","ts":"2026-07-13T15:49:07.341Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37ec5d9e-1d2d-49f8-b7b8-62438f36379c","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:07.345Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"c9af65b8-885c-4fc9-b2c8-5d88efe32fa7","name":"raycluster-head"}
{"level":"info","ts":"2026-07-13T15:49:07.347Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"344afef0-aa4d-4a27-9afb-9c1b3762c94d"}
{"level":"info","ts":"2026-07-13T15:49:07.347Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"344afef0-aa4d-4a27-9afb-9c1b3762c94d"}
{"level":"info","ts":"2026-07-13T15:49:07.347Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"344afef0-aa4d-4a27-9afb-9c1b3762c94d","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:07.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"344afef0-aa4d-4a27-9afb-9c1b3762c94d","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:07.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"344afef0-aa4d-4a27-9afb-9c1b3762c94d","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:07.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"344afef0-aa4d-4a27-9afb-9c1b3762c94d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:07.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"344afef0-aa4d-4a27-9afb-9c1b3762c94d","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:07.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"344afef0-aa4d-4a27-9afb-9c1b3762c94d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:07.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"344afef0-aa4d-4a27-9afb-9c1b3762c94d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:07.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"344afef0-aa4d-4a27-9afb-9c1b3762c94d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:07.347Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"344afef0-aa4d-4a27-9afb-9c1b3762c94d","podName":"raycluster-head-fdvcn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:07.347Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"344afef0-aa4d-4a27-9afb-9c1b3762c94d","podName":"raycluster-head-fdvcn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:07.347Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"344afef0-aa4d-4a27-9afb-9c1b3762c94d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:07.347Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"344afef0-aa4d-4a27-9afb-9c1b3762c94d","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:07.354Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"c9af65b8-885c-4fc9-b2c8-5d88efe32fa7","name":"raycluster-workers"}
{"level":"info","ts":"2026-07-13T15:49:07.354Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"c9af65b8-885c-4fc9-b2c8-5d88efe32fa7","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:07.696Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ccfe5224-206e-41f6-baf1-736e7f01078c"}
{"level":"info","ts":"2026-07-13T15:49:07.696Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ccfe5224-206e-41f6-baf1-736e7f01078c"}
{"level":"info","ts":"2026-07-13T15:49:07.696Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ccfe5224-206e-41f6-baf1-736e7f01078c","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:07.696Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ccfe5224-206e-41f6-baf1-736e7f01078c","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:07.696Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ccfe5224-206e-41f6-baf1-736e7f01078c","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:07.696Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ccfe5224-206e-41f6-baf1-736e7f01078c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:07.696Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ccfe5224-206e-41f6-baf1-736e7f01078c","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:07.696Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ccfe5224-206e-41f6-baf1-736e7f01078c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:07.696Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ccfe5224-206e-41f6-baf1-736e7f01078c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:07.696Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ccfe5224-206e-41f6-baf1-736e7f01078c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:07.697Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ccfe5224-206e-41f6-baf1-736e7f01078c","podName":"raycluster-small-group-worker-96vng","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:07.697Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ccfe5224-206e-41f6-baf1-736e7f01078c","podName":"raycluster-small-group-worker-96vng","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:07.697Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ccfe5224-206e-41f6-baf1-736e7f01078c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:07.697Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ccfe5224-206e-41f6-baf1-736e7f01078c","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:07.707Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"adb58db8-9907-459e-a739-88dd1fcb5be9"}
{"level":"info","ts":"2026-07-13T15:49:07.707Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"adb58db8-9907-459e-a739-88dd1fcb5be9"}
{"level":"info","ts":"2026-07-13T15:49:07.707Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"adb58db8-9907-459e-a739-88dd1fcb5be9","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:07.707Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"adb58db8-9907-459e-a739-88dd1fcb5be9","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:07.707Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"adb58db8-9907-459e-a739-88dd1fcb5be9","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:07.707Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"adb58db8-9907-459e-a739-88dd1fcb5be9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:07.707Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"adb58db8-9907-459e-a739-88dd1fcb5be9","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:07.707Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"adb58db8-9907-459e-a739-88dd1fcb5be9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:07.707Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"adb58db8-9907-459e-a739-88dd1fcb5be9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:07.707Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"adb58db8-9907-459e-a739-88dd1fcb5be9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:07.707Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"adb58db8-9907-459e-a739-88dd1fcb5be9","podName":"raycluster-head-fdvcn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:07.707Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"adb58db8-9907-459e-a739-88dd1fcb5be9","podName":"raycluster-head-fdvcn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:07.707Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"adb58db8-9907-459e-a739-88dd1fcb5be9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:07.707Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"adb58db8-9907-459e-a739-88dd1fcb5be9","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:08.682Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2b04a972-5494-4603-92bb-ce0f3ebbe95f"}
{"level":"info","ts":"2026-07-13T15:49:08.682Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2b04a972-5494-4603-92bb-ce0f3ebbe95f"}
{"level":"info","ts":"2026-07-13T15:49:08.682Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2b04a972-5494-4603-92bb-ce0f3ebbe95f","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:08.683Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2b04a972-5494-4603-92bb-ce0f3ebbe95f","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:08.683Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2b04a972-5494-4603-92bb-ce0f3ebbe95f","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:08.683Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2b04a972-5494-4603-92bb-ce0f3ebbe95f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:08.683Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2b04a972-5494-4603-92bb-ce0f3ebbe95f","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:08.683Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2b04a972-5494-4603-92bb-ce0f3ebbe95f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:08.683Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2b04a972-5494-4603-92bb-ce0f3ebbe95f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:08.683Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2b04a972-5494-4603-92bb-ce0f3ebbe95f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:08.683Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2b04a972-5494-4603-92bb-ce0f3ebbe95f","podName":"raycluster-small-group-worker-96vng","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:08.683Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2b04a972-5494-4603-92bb-ce0f3ebbe95f","podName":"raycluster-small-group-worker-96vng","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:08.683Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2b04a972-5494-4603-92bb-ce0f3ebbe95f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:08.683Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2b04a972-5494-4603-92bb-ce0f3ebbe95f","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:08.777Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"426389f5-304c-4cdd-ae1d-8f8abd6e1f69"}
{"level":"info","ts":"2026-07-13T15:49:08.777Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"426389f5-304c-4cdd-ae1d-8f8abd6e1f69"}
{"level":"info","ts":"2026-07-13T15:49:08.777Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"426389f5-304c-4cdd-ae1d-8f8abd6e1f69","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:08.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"426389f5-304c-4cdd-ae1d-8f8abd6e1f69","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:08.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"426389f5-304c-4cdd-ae1d-8f8abd6e1f69","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:08.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"426389f5-304c-4cdd-ae1d-8f8abd6e1f69","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:08.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"426389f5-304c-4cdd-ae1d-8f8abd6e1f69","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:08.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"426389f5-304c-4cdd-ae1d-8f8abd6e1f69","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:08.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"426389f5-304c-4cdd-ae1d-8f8abd6e1f69","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:08.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"426389f5-304c-4cdd-ae1d-8f8abd6e1f69","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:08.777Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"426389f5-304c-4cdd-ae1d-8f8abd6e1f69","podName":"raycluster-head-fdvcn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:08.777Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"426389f5-304c-4cdd-ae1d-8f8abd6e1f69","podName":"raycluster-head-fdvcn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:08.777Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"426389f5-304c-4cdd-ae1d-8f8abd6e1f69","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"newHead":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"}}
{"level":"info","ts":"2026-07-13T15:49:08.777Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"426389f5-304c-4cdd-ae1d-8f8abd6e1f69","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:08.788Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"d778dd66-f052-4f7e-8047-624b9cae036e","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:08.788Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"ff39a0f4-1284-40d6-ba6c-73216571633b","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:08.788Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"d778dd66-f052-4f7e-8047-624b9cae036e","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T15:49:08.788Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"d778dd66-f052-4f7e-8047-624b9cae036e","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T15:49:08.800Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"d778dd66-f052-4f7e-8047-624b9cae036e","certificate":"ray-head-cert-raycluster","dnsNames":["raycluster-head-svc","localhost","raycluster-head-svc.test-ns-qbjjb.svc","raycluster-head-svc.test-ns-qbjjb.svc.cluster.local"],"ipAddresses":["10.134.0.25","10.133.0.18","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:49:08.805Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"ff39a0f4-1284-40d6-ba6c-73216571633b","name":"raycluster-head"}
{"level":"info","ts":"2026-07-13T15:49:08.809Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"d778dd66-f052-4f7e-8047-624b9cae036e","certificate":"ray-worker-cert-raycluster","dnsNames":["raycluster-worker-svc","localhost","raycluster-small-group","raycluster-worker-svc.test-ns-qbjjb.svc","raycluster-worker-svc.test-ns-qbjjb.svc.cluster.local","*.raycluster-worker-svc.test-ns-qbjjb.svc","*.raycluster-worker-svc.test-ns-qbjjb.svc.cluster.local","*-worker-*.test-ns-qbjjb.svc","*-worker-*.test-ns-qbjjb.svc.cluster.local"],"ipAddresses":["10.134.0.25","10.133.0.18","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:49:08.809Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"d778dd66-f052-4f7e-8047-624b9cae036e","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T15:49:08.820Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"ff39a0f4-1284-40d6-ba6c-73216571633b","name":"raycluster-workers"}
{"level":"info","ts":"2026-07-13T15:49:08.821Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"ff39a0f4-1284-40d6-ba6c-73216571633b","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:09.521Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e2403a42-ea51-4842-8923-2273e28ef84d"}
{"level":"info","ts":"2026-07-13T15:49:09.521Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e2403a42-ea51-4842-8923-2273e28ef84d"}
{"level":"info","ts":"2026-07-13T15:49:09.521Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e2403a42-ea51-4842-8923-2273e28ef84d","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:09.521Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e2403a42-ea51-4842-8923-2273e28ef84d","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:09.521Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e2403a42-ea51-4842-8923-2273e28ef84d","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:09.521Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e2403a42-ea51-4842-8923-2273e28ef84d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:09.521Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e2403a42-ea51-4842-8923-2273e28ef84d","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:09.521Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e2403a42-ea51-4842-8923-2273e28ef84d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:09.521Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e2403a42-ea51-4842-8923-2273e28ef84d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:09.521Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e2403a42-ea51-4842-8923-2273e28ef84d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:09.522Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e2403a42-ea51-4842-8923-2273e28ef84d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:09.522Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e2403a42-ea51-4842-8923-2273e28ef84d","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:09.525Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"423065d8-d448-498b-9ade-a2c5791c796f"}
{"level":"info","ts":"2026-07-13T15:49:09.525Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"423065d8-d448-498b-9ade-a2c5791c796f"}
{"level":"info","ts":"2026-07-13T15:49:09.525Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"423065d8-d448-498b-9ade-a2c5791c796f","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:09.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"423065d8-d448-498b-9ade-a2c5791c796f","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:09.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"423065d8-d448-498b-9ade-a2c5791c796f","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:09.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"423065d8-d448-498b-9ade-a2c5791c796f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:09.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"423065d8-d448-498b-9ade-a2c5791c796f","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:09.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"423065d8-d448-498b-9ade-a2c5791c796f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:09.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"423065d8-d448-498b-9ade-a2c5791c796f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:09.526Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"423065d8-d448-498b-9ade-a2c5791c796f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:09.526Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"423065d8-d448-498b-9ade-a2c5791c796f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:09.526Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"423065d8-d448-498b-9ade-a2c5791c796f","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:09.542Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0650a947-521a-442b-8c5b-7d15a8f50ea7"}
{"level":"info","ts":"2026-07-13T15:49:09.542Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0650a947-521a-442b-8c5b-7d15a8f50ea7"}
{"level":"info","ts":"2026-07-13T15:49:09.542Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0650a947-521a-442b-8c5b-7d15a8f50ea7","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:09.543Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0650a947-521a-442b-8c5b-7d15a8f50ea7","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:09.543Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0650a947-521a-442b-8c5b-7d15a8f50ea7","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:09.543Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0650a947-521a-442b-8c5b-7d15a8f50ea7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:09.543Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0650a947-521a-442b-8c5b-7d15a8f50ea7","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:09.543Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0650a947-521a-442b-8c5b-7d15a8f50ea7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:09.543Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0650a947-521a-442b-8c5b-7d15a8f50ea7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:09.543Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0650a947-521a-442b-8c5b-7d15a8f50ea7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:09.543Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0650a947-521a-442b-8c5b-7d15a8f50ea7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:09.543Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0650a947-521a-442b-8c5b-7d15a8f50ea7","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:09.545Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"314fb50a-d4de-413a-8681-cf08d75b90e2"}
{"level":"info","ts":"2026-07-13T15:49:09.545Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"314fb50a-d4de-413a-8681-cf08d75b90e2"}
{"level":"info","ts":"2026-07-13T15:49:09.545Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"314fb50a-d4de-413a-8681-cf08d75b90e2","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:09.545Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"314fb50a-d4de-413a-8681-cf08d75b90e2","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:09.545Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"314fb50a-d4de-413a-8681-cf08d75b90e2","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:09.545Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"314fb50a-d4de-413a-8681-cf08d75b90e2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:09.546Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"314fb50a-d4de-413a-8681-cf08d75b90e2","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-z4nbk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:09.546Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"314fb50a-d4de-413a-8681-cf08d75b90e2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:09.546Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"314fb50a-d4de-413a-8681-cf08d75b90e2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:09.546Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"314fb50a-d4de-413a-8681-cf08d75b90e2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:09.546Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"314fb50a-d4de-413a-8681-cf08d75b90e2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:09.546Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"314fb50a-d4de-413a-8681-cf08d75b90e2","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:10.085Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d19ac1b2-ff01-487b-8340-aff92d99a38b"}
{"level":"info","ts":"2026-07-13T15:49:10.085Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d19ac1b2-ff01-487b-8340-aff92d99a38b"}
{"level":"info","ts":"2026-07-13T15:49:10.085Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d19ac1b2-ff01-487b-8340-aff92d99a38b","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:10.085Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d19ac1b2-ff01-487b-8340-aff92d99a38b","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:10.085Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d19ac1b2-ff01-487b-8340-aff92d99a38b","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:10.085Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d19ac1b2-ff01-487b-8340-aff92d99a38b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:10.085Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d19ac1b2-ff01-487b-8340-aff92d99a38b","worker Pod":"raycluster-auth-small-group-worker-z4nbk","shouldDelete":true,"reason":"The worker Pod raycluster-auth-small-group-worker-z4nbk status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-07-13T15:49:10.089Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d19ac1b2-ff01-487b-8340-aff92d99a38b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"delete 1 unhealthy worker Pods","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:10.089Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d19ac1b2-ff01-487b-8340-aff92d99a38b","error":"delete 1 unhealthy worker Pods"}
{"level":"info","ts":"2026-07-13T15:49:10.089Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d19ac1b2-ff01-487b-8340-aff92d99a38b","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:49:10.089Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d19ac1b2-ff01-487b-8340-aff92d99a38b","name":"raycluster-auth","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:03Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:03Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:47:39Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:03Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:10Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:03Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:47:39Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:03Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:10.101Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d19ac1b2-ff01-487b-8340-aff92d99a38b"}
{"level":"error","ts":"2026-07-13T15:49:10.101Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d19ac1b2-ff01-487b-8340-aff92d99a38b","error":"delete 1 unhealthy worker Pods","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:10.101Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"72349427-fa51-44fc-bf9c-6618196bd6c3","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:10.101Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"9dc84af1-7cd2-4803-88d2-f86f569e5fa0","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:10.101Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"72349427-fa51-44fc-bf9c-6618196bd6c3","pod":"raycluster-auth-head-czcms","ip":"10.133.0.17"}
{"level":"info","ts":"2026-07-13T15:49:10.101Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"72349427-fa51-44fc-bf9c-6618196bd6c3","pod":"raycluster-auth-small-group-worker-z4nbk","ip":"10.134.0.24"}
{"level":"info","ts":"2026-07-13T15:49:10.101Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"72349427-fa51-44fc-bf9c-6618196bd6c3","rayCluster":"raycluster-auth","podIPs":["10.133.0.17","10.134.0.24"]}
{"level":"info","ts":"2026-07-13T15:49:10.106Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"9dc84af1-7cd2-4803-88d2-f86f569e5fa0","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:10.106Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1bdb6f24-fbd3-4b97-86cb-79266e30f9ef"}
{"level":"info","ts":"2026-07-13T15:49:10.106Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1bdb6f24-fbd3-4b97-86cb-79266e30f9ef"}
{"level":"info","ts":"2026-07-13T15:49:10.106Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1bdb6f24-fbd3-4b97-86cb-79266e30f9ef","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:10.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1bdb6f24-fbd3-4b97-86cb-79266e30f9ef","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:10.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1bdb6f24-fbd3-4b97-86cb-79266e30f9ef","head Pod":"raycluster-auth-head-czcms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-czcms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:10.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1bdb6f24-fbd3-4b97-86cb-79266e30f9ef","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T15:49:10.106Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1bdb6f24-fbd3-4b97-86cb-79266e30f9ef","podName":"raycluster-auth-small-group-worker-z4nbk","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-13T15:49:10.106Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1bdb6f24-fbd3-4b97-86cb-79266e30f9ef","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:10.106Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1bdb6f24-fbd3-4b97-86cb-79266e30f9ef","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:10.280Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1cf79c38-cc10-4bdf-a726-406d2b3983ea"}
{"level":"info","ts":"2026-07-13T15:49:10.280Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1cf79c38-cc10-4bdf-a726-406d2b3983ea"}
{"level":"info","ts":"2026-07-13T15:49:10.280Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1cf79c38-cc10-4bdf-a726-406d2b3983ea","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:10.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1cf79c38-cc10-4bdf-a726-406d2b3983ea","Found 1 head Pod":"raycluster-auth-head-czcms","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-07-13 15:47:25 +0000 UTC,FinishedAt:2026-07-13 15:49:10 +0000 UTC,ContainerID:cri-o://705cd47c622c878bdbf2f2458cbf7357f43bbb03e08f5f7e1ac21f22838ae9ac,}"}
{"level":"info","ts":"2026-07-13T15:49:10.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1cf79c38-cc10-4bdf-a726-406d2b3983ea","head Pod":"raycluster-auth-head-czcms","shouldDelete":true,"reason":"The head Pod raycluster-auth-head-czcms status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-07-13T15:49:10.283Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1cf79c38-cc10-4bdf-a726-406d2b3983ea","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod raycluster-auth-head-czcms status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:10.283Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1cf79c38-cc10-4bdf-a726-406d2b3983ea","error":"The head Pod raycluster-auth-head-czcms status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-07-13T15:49:10.284Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1cf79c38-cc10-4bdf-a726-406d2b3983ea","old conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:47:39Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:03Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:10Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:03Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:49:10.284Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1cf79c38-cc10-4bdf-a726-406d2b3983ea","name":"raycluster-auth","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:10Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:03Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:47:39Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:03Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:10Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:03Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:10Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:03Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:10.294Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1cf79c38-cc10-4bdf-a726-406d2b3983ea"}
{"level":"error","ts":"2026-07-13T15:49:10.294Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1cf79c38-cc10-4bdf-a726-406d2b3983ea","error":"The head Pod raycluster-auth-head-czcms status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:10.295Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"19670915-6744-4093-8f27-1f5fcde23deb","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:10.295Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"26011c4b-43ef-4deb-bdc5-8c1e077f4307","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:10.295Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"26011c4b-43ef-4deb-bdc5-8c1e077f4307","pod":"raycluster-auth-head-czcms","ip":"10.133.0.17"}
{"level":"info","ts":"2026-07-13T15:49:10.295Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"26011c4b-43ef-4deb-bdc5-8c1e077f4307","pod":"raycluster-auth-small-group-worker-z4nbk","ip":"10.134.0.24"}
{"level":"info","ts":"2026-07-13T15:49:10.295Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"26011c4b-43ef-4deb-bdc5-8c1e077f4307","rayCluster":"raycluster-auth","podIPs":["10.133.0.17","10.134.0.24"]}
{"level":"info","ts":"2026-07-13T15:49:10.299Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"19670915-6744-4093-8f27-1f5fcde23deb","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:10.300Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3fc94ed-7dea-463a-b861-065cd22199b9"}
{"level":"info","ts":"2026-07-13T15:49:10.300Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3fc94ed-7dea-463a-b861-065cd22199b9"}
{"level":"info","ts":"2026-07-13T15:49:10.300Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3fc94ed-7dea-463a-b861-065cd22199b9","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:10.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3fc94ed-7dea-463a-b861-065cd22199b9","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-13T15:49:10.301Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3fc94ed-7dea-463a-b861-065cd22199b9","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T15:49:10.301Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3fc94ed-7dea-463a-b861-065cd22199b9","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-13T15:49:10.301Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3fc94ed-7dea-463a-b861-065cd22199b9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:10.301Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3fc94ed-7dea-463a-b861-065cd22199b9","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:10.700Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"9cb924c3-b2e0-40c9-8860-7ea3889316f8"}
{"level":"info","ts":"2026-07-13T15:49:10.700Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"9cb924c3-b2e0-40c9-8860-7ea3889316f8"}
{"level":"info","ts":"2026-07-13T15:49:10.700Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"9cb924c3-b2e0-40c9-8860-7ea3889316f8","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:10.700Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"9cb924c3-b2e0-40c9-8860-7ea3889316f8","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-13T15:49:10.700Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"9cb924c3-b2e0-40c9-8860-7ea3889316f8","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T15:49:10.700Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"9cb924c3-b2e0-40c9-8860-7ea3889316f8","podName":"raycluster-auth-head-czcms","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-13T15:49:10.700Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"9cb924c3-b2e0-40c9-8860-7ea3889316f8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:10.700Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"9cb924c3-b2e0-40c9-8860-7ea3889316f8","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:10.704Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1"}
{"level":"info","ts":"2026-07-13T15:49:10.704Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1"}
{"level":"info","ts":"2026-07-13T15:49:10.704Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:10.704Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1"}
{"level":"info","ts":"2026-07-13T15:49:10.704Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:10.704Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1"}
{"level":"info","ts":"2026-07-13T15:49:10.704Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:10.704Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:10.704Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T15:49:10.704Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T15:49:10.704Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T15:49:10.704Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:10.704Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:10.704Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1","enabled":true}
{"level":"error","ts":"2026-07-13T15:49:10.708Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:10.708Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:49:10.708Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.708Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.708Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.708Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"newHead":{"serviceName":"raycluster-auth-head-svc"}}
{"level":"info","ts":"2026-07-13T15:49:10.708Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1","name":"raycluster-auth","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:10Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:03Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-czcms","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:10Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:03Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:10Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:03Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:03Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:43:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:43:26Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T15:49:10Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:10.719Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1"}
{"level":"error","ts":"2026-07-13T15:49:10.719Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"0403bc6b-bda9-4441-afaa-0ad7e0ab9aa1","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:10.720Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e8e46e21-1feb-44fb-818e-ffc453a3b339","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:10.720Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"daecaf02-9fcc-4840-95d4-cb01aff8d1b8","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:10.720Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e8e46e21-1feb-44fb-818e-ffc453a3b339","pod":"raycluster-auth-small-group-worker-z4nbk","ip":"10.134.0.24"}
{"level":"info","ts":"2026-07-13T15:49:10.724Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"daecaf02-9fcc-4840-95d4-cb01aff8d1b8","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:10.725Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f7e1b82c-3205-4acd-81d2-29641194c1fe"}
{"level":"info","ts":"2026-07-13T15:49:10.725Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f7e1b82c-3205-4acd-81d2-29641194c1fe"}
{"level":"info","ts":"2026-07-13T15:49:10.725Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f7e1b82c-3205-4acd-81d2-29641194c1fe","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:10.725Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f7e1b82c-3205-4acd-81d2-29641194c1fe"}
{"level":"info","ts":"2026-07-13T15:49:10.725Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f7e1b82c-3205-4acd-81d2-29641194c1fe","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:10.725Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f7e1b82c-3205-4acd-81d2-29641194c1fe"}
{"level":"info","ts":"2026-07-13T15:49:10.725Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f7e1b82c-3205-4acd-81d2-29641194c1fe","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:10.725Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f7e1b82c-3205-4acd-81d2-29641194c1fe","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:10.725Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f7e1b82c-3205-4acd-81d2-29641194c1fe","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T15:49:10.725Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f7e1b82c-3205-4acd-81d2-29641194c1fe","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T15:49:10.725Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f7e1b82c-3205-4acd-81d2-29641194c1fe","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T15:49:10.725Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f7e1b82c-3205-4acd-81d2-29641194c1fe","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:10.725Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f7e1b82c-3205-4acd-81d2-29641194c1fe","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:10.725Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f7e1b82c-3205-4acd-81d2-29641194c1fe","enabled":true}
{"level":"error","ts":"2026-07-13T15:49:10.728Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f7e1b82c-3205-4acd-81d2-29641194c1fe","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:10.729Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f7e1b82c-3205-4acd-81d2-29641194c1fe","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:49:10.729Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f7e1b82c-3205-4acd-81d2-29641194c1fe","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.729Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f7e1b82c-3205-4acd-81d2-29641194c1fe","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.729Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f7e1b82c-3205-4acd-81d2-29641194c1fe","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.729Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f7e1b82c-3205-4acd-81d2-29641194c1fe"}
{"level":"error","ts":"2026-07-13T15:49:10.729Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"f7e1b82c-3205-4acd-81d2-29641194c1fe","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:10.730Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e8e46e21-1feb-44fb-818e-ffc453a3b339","certificate":"ray-head-cert-raycluster-auth","dnsNames":["raycluster-auth-head-svc","localhost","raycluster-auth-head-svc.test-ns-k9mkf.svc","raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local"],"ipAddresses":["10.134.0.24","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:49:10.740Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6b7a3c37-94cc-4705-842f-5be3f7473886"}
{"level":"info","ts":"2026-07-13T15:49:10.740Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6b7a3c37-94cc-4705-842f-5be3f7473886"}
{"level":"info","ts":"2026-07-13T15:49:10.740Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6b7a3c37-94cc-4705-842f-5be3f7473886","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:10.740Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6b7a3c37-94cc-4705-842f-5be3f7473886"}
{"level":"info","ts":"2026-07-13T15:49:10.740Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6b7a3c37-94cc-4705-842f-5be3f7473886","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:10.740Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6b7a3c37-94cc-4705-842f-5be3f7473886"}
{"level":"info","ts":"2026-07-13T15:49:10.740Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6b7a3c37-94cc-4705-842f-5be3f7473886","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:10.740Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6b7a3c37-94cc-4705-842f-5be3f7473886","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:10.740Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6b7a3c37-94cc-4705-842f-5be3f7473886","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T15:49:10.740Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6b7a3c37-94cc-4705-842f-5be3f7473886","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T15:49:10.740Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6b7a3c37-94cc-4705-842f-5be3f7473886","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T15:49:10.740Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6b7a3c37-94cc-4705-842f-5be3f7473886","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:10.740Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6b7a3c37-94cc-4705-842f-5be3f7473886","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:10.740Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6b7a3c37-94cc-4705-842f-5be3f7473886","enabled":true}
{"level":"info","ts":"2026-07-13T15:49:10.743Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e8e46e21-1feb-44fb-818e-ffc453a3b339","certificate":"ray-worker-cert-raycluster-auth","dnsNames":["raycluster-auth-worker-svc","localhost","raycluster-auth-small-group","raycluster-auth-worker-svc.test-ns-k9mkf.svc","raycluster-auth-worker-svc.test-ns-k9mkf.svc.cluster.local","*.raycluster-auth-worker-svc.test-ns-k9mkf.svc","*.raycluster-auth-worker-svc.test-ns-k9mkf.svc.cluster.local","*-worker-*.test-ns-k9mkf.svc","*-worker-*.test-ns-k9mkf.svc.cluster.local"],"ipAddresses":["10.134.0.24","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:49:10.743Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e8e46e21-1feb-44fb-818e-ffc453a3b339","rayCluster":"raycluster-auth","podIPs":["10.134.0.24"]}
{"level":"error","ts":"2026-07-13T15:49:10.743Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6b7a3c37-94cc-4705-842f-5be3f7473886","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:10.744Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6b7a3c37-94cc-4705-842f-5be3f7473886","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:49:10.744Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6b7a3c37-94cc-4705-842f-5be3f7473886","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.744Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6b7a3c37-94cc-4705-842f-5be3f7473886","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.744Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6b7a3c37-94cc-4705-842f-5be3f7473886","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.744Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6b7a3c37-94cc-4705-842f-5be3f7473886"}
{"level":"error","ts":"2026-07-13T15:49:10.744Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"6b7a3c37-94cc-4705-842f-5be3f7473886","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:10.764Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c385a63b-95a5-4f54-a5cb-402c5479cc29"}
{"level":"info","ts":"2026-07-13T15:49:10.764Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c385a63b-95a5-4f54-a5cb-402c5479cc29"}
{"level":"info","ts":"2026-07-13T15:49:10.764Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c385a63b-95a5-4f54-a5cb-402c5479cc29","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:10.764Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c385a63b-95a5-4f54-a5cb-402c5479cc29"}
{"level":"info","ts":"2026-07-13T15:49:10.764Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c385a63b-95a5-4f54-a5cb-402c5479cc29","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:10.764Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c385a63b-95a5-4f54-a5cb-402c5479cc29"}
{"level":"info","ts":"2026-07-13T15:49:10.765Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c385a63b-95a5-4f54-a5cb-402c5479cc29","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:10.765Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c385a63b-95a5-4f54-a5cb-402c5479cc29","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:10.765Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c385a63b-95a5-4f54-a5cb-402c5479cc29","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T15:49:10.765Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c385a63b-95a5-4f54-a5cb-402c5479cc29","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T15:49:10.765Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c385a63b-95a5-4f54-a5cb-402c5479cc29","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T15:49:10.765Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c385a63b-95a5-4f54-a5cb-402c5479cc29","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:10.765Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c385a63b-95a5-4f54-a5cb-402c5479cc29","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:10.765Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c385a63b-95a5-4f54-a5cb-402c5479cc29","enabled":true}
{"level":"error","ts":"2026-07-13T15:49:10.768Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c385a63b-95a5-4f54-a5cb-402c5479cc29","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:10.768Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c385a63b-95a5-4f54-a5cb-402c5479cc29","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:49:10.768Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c385a63b-95a5-4f54-a5cb-402c5479cc29","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.768Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c385a63b-95a5-4f54-a5cb-402c5479cc29","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.768Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c385a63b-95a5-4f54-a5cb-402c5479cc29","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.768Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c385a63b-95a5-4f54-a5cb-402c5479cc29"}
{"level":"error","ts":"2026-07-13T15:49:10.768Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"c385a63b-95a5-4f54-a5cb-402c5479cc29","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:10.783Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0dd0dfe-44b3-4ea4-bc13-b50f2f61b962"}
{"level":"info","ts":"2026-07-13T15:49:10.783Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0dd0dfe-44b3-4ea4-bc13-b50f2f61b962"}
{"level":"info","ts":"2026-07-13T15:49:10.783Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0dd0dfe-44b3-4ea4-bc13-b50f2f61b962","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:10.784Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0dd0dfe-44b3-4ea4-bc13-b50f2f61b962"}
{"level":"info","ts":"2026-07-13T15:49:10.784Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0dd0dfe-44b3-4ea4-bc13-b50f2f61b962","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:10.784Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0dd0dfe-44b3-4ea4-bc13-b50f2f61b962"}
{"level":"info","ts":"2026-07-13T15:49:10.784Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0dd0dfe-44b3-4ea4-bc13-b50f2f61b962","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:10.784Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0dd0dfe-44b3-4ea4-bc13-b50f2f61b962","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:10.784Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0dd0dfe-44b3-4ea4-bc13-b50f2f61b962","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T15:49:10.784Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0dd0dfe-44b3-4ea4-bc13-b50f2f61b962","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T15:49:10.784Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0dd0dfe-44b3-4ea4-bc13-b50f2f61b962","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T15:49:10.784Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0dd0dfe-44b3-4ea4-bc13-b50f2f61b962","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:10.784Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0dd0dfe-44b3-4ea4-bc13-b50f2f61b962","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:10.784Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0dd0dfe-44b3-4ea4-bc13-b50f2f61b962","enabled":true}
{"level":"error","ts":"2026-07-13T15:49:10.786Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0dd0dfe-44b3-4ea4-bc13-b50f2f61b962","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:10.786Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0dd0dfe-44b3-4ea4-bc13-b50f2f61b962","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:49:10.787Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0dd0dfe-44b3-4ea4-bc13-b50f2f61b962","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.787Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0dd0dfe-44b3-4ea4-bc13-b50f2f61b962","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.787Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0dd0dfe-44b3-4ea4-bc13-b50f2f61b962","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.787Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0dd0dfe-44b3-4ea4-bc13-b50f2f61b962"}
{"level":"error","ts":"2026-07-13T15:49:10.787Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"d0dd0dfe-44b3-4ea4-bc13-b50f2f61b962","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:10.787Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3b08431-834b-4a52-998e-2dbe9f327f89"}
{"level":"info","ts":"2026-07-13T15:49:10.787Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3b08431-834b-4a52-998e-2dbe9f327f89"}
{"level":"info","ts":"2026-07-13T15:49:10.787Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3b08431-834b-4a52-998e-2dbe9f327f89","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:10.787Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3b08431-834b-4a52-998e-2dbe9f327f89"}
{"level":"info","ts":"2026-07-13T15:49:10.787Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3b08431-834b-4a52-998e-2dbe9f327f89","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:10.787Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3b08431-834b-4a52-998e-2dbe9f327f89"}
{"level":"info","ts":"2026-07-13T15:49:10.787Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3b08431-834b-4a52-998e-2dbe9f327f89","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:10.787Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3b08431-834b-4a52-998e-2dbe9f327f89","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:10.787Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3b08431-834b-4a52-998e-2dbe9f327f89","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T15:49:10.787Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3b08431-834b-4a52-998e-2dbe9f327f89","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T15:49:10.787Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3b08431-834b-4a52-998e-2dbe9f327f89","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T15:49:10.788Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3b08431-834b-4a52-998e-2dbe9f327f89","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:10.788Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3b08431-834b-4a52-998e-2dbe9f327f89","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:10.788Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3b08431-834b-4a52-998e-2dbe9f327f89","enabled":true}
{"level":"error","ts":"2026-07-13T15:49:10.821Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3b08431-834b-4a52-998e-2dbe9f327f89","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:10.821Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3b08431-834b-4a52-998e-2dbe9f327f89","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:49:10.821Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3b08431-834b-4a52-998e-2dbe9f327f89","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.821Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3b08431-834b-4a52-998e-2dbe9f327f89","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.821Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3b08431-834b-4a52-998e-2dbe9f327f89","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.821Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3b08431-834b-4a52-998e-2dbe9f327f89"}
{"level":"error","ts":"2026-07-13T15:49:10.821Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"e3b08431-834b-4a52-998e-2dbe9f327f89","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:10.821Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fe064903-0bb9-40ce-8e8c-cc5322bc410b"}
{"level":"info","ts":"2026-07-13T15:49:10.821Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fe064903-0bb9-40ce-8e8c-cc5322bc410b"}
{"level":"info","ts":"2026-07-13T15:49:10.821Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fe064903-0bb9-40ce-8e8c-cc5322bc410b","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:10.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fe064903-0bb9-40ce-8e8c-cc5322bc410b","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:10.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fe064903-0bb9-40ce-8e8c-cc5322bc410b","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:10.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fe064903-0bb9-40ce-8e8c-cc5322bc410b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:10.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fe064903-0bb9-40ce-8e8c-cc5322bc410b","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:10.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fe064903-0bb9-40ce-8e8c-cc5322bc410b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:10.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fe064903-0bb9-40ce-8e8c-cc5322bc410b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:10.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fe064903-0bb9-40ce-8e8c-cc5322bc410b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:10.821Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fe064903-0bb9-40ce-8e8c-cc5322bc410b","podName":"raycluster-head-fdvcn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:10.821Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fe064903-0bb9-40ce-8e8c-cc5322bc410b","podName":"raycluster-head-fdvcn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:10.822Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fe064903-0bb9-40ce-8e8c-cc5322bc410b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:10.822Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fe064903-0bb9-40ce-8e8c-cc5322bc410b","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:10.822Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d922a52-4152-457f-a9b1-fe2b8a94fea1"}
{"level":"info","ts":"2026-07-13T15:49:10.822Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d922a52-4152-457f-a9b1-fe2b8a94fea1"}
{"level":"info","ts":"2026-07-13T15:49:10.822Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d922a52-4152-457f-a9b1-fe2b8a94fea1","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:10.822Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d922a52-4152-457f-a9b1-fe2b8a94fea1"}
{"level":"info","ts":"2026-07-13T15:49:10.822Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d922a52-4152-457f-a9b1-fe2b8a94fea1","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:10.822Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d922a52-4152-457f-a9b1-fe2b8a94fea1"}
{"level":"info","ts":"2026-07-13T15:49:10.822Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d922a52-4152-457f-a9b1-fe2b8a94fea1","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:10.822Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d922a52-4152-457f-a9b1-fe2b8a94fea1","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:10.822Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d922a52-4152-457f-a9b1-fe2b8a94fea1","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T15:49:10.822Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d922a52-4152-457f-a9b1-fe2b8a94fea1","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T15:49:10.822Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d922a52-4152-457f-a9b1-fe2b8a94fea1","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T15:49:10.822Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d922a52-4152-457f-a9b1-fe2b8a94fea1","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:10.822Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d922a52-4152-457f-a9b1-fe2b8a94fea1","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:10.822Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d922a52-4152-457f-a9b1-fe2b8a94fea1","enabled":true}
{"level":"error","ts":"2026-07-13T15:49:10.825Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d922a52-4152-457f-a9b1-fe2b8a94fea1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:10.825Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d922a52-4152-457f-a9b1-fe2b8a94fea1","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:49:10.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d922a52-4152-457f-a9b1-fe2b8a94fea1","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d922a52-4152-457f-a9b1-fe2b8a94fea1","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d922a52-4152-457f-a9b1-fe2b8a94fea1","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.825Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d922a52-4152-457f-a9b1-fe2b8a94fea1"}
{"level":"error","ts":"2026-07-13T15:49:10.825Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1d922a52-4152-457f-a9b1-fe2b8a94fea1","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:10.982Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"12f63df6-dba0-40f8-92a2-bceb541a00b6"}
{"level":"info","ts":"2026-07-13T15:49:10.982Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"12f63df6-dba0-40f8-92a2-bceb541a00b6"}
{"level":"info","ts":"2026-07-13T15:49:10.982Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"12f63df6-dba0-40f8-92a2-bceb541a00b6","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:10.982Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"12f63df6-dba0-40f8-92a2-bceb541a00b6"}
{"level":"info","ts":"2026-07-13T15:49:10.982Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"12f63df6-dba0-40f8-92a2-bceb541a00b6","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:10.982Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"12f63df6-dba0-40f8-92a2-bceb541a00b6"}
{"level":"info","ts":"2026-07-13T15:49:10.982Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"12f63df6-dba0-40f8-92a2-bceb541a00b6","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:10.982Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"12f63df6-dba0-40f8-92a2-bceb541a00b6","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:10.982Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"12f63df6-dba0-40f8-92a2-bceb541a00b6","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T15:49:10.982Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"12f63df6-dba0-40f8-92a2-bceb541a00b6","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T15:49:10.982Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"12f63df6-dba0-40f8-92a2-bceb541a00b6","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T15:49:10.982Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"12f63df6-dba0-40f8-92a2-bceb541a00b6","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:10.982Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"12f63df6-dba0-40f8-92a2-bceb541a00b6","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:10.982Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"12f63df6-dba0-40f8-92a2-bceb541a00b6","enabled":true}
{"level":"error","ts":"2026-07-13T15:49:10.986Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"12f63df6-dba0-40f8-92a2-bceb541a00b6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:10.986Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"12f63df6-dba0-40f8-92a2-bceb541a00b6","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:49:10.986Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"12f63df6-dba0-40f8-92a2-bceb541a00b6","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.986Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"12f63df6-dba0-40f8-92a2-bceb541a00b6","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.986Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"12f63df6-dba0-40f8-92a2-bceb541a00b6","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:10.986Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"12f63df6-dba0-40f8-92a2-bceb541a00b6"}
{"level":"error","ts":"2026-07-13T15:49:10.986Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"12f63df6-dba0-40f8-92a2-bceb541a00b6","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:11.627Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"239e60a0-6a0a-4c1f-90f3-f3779bc65ff4"}
{"level":"info","ts":"2026-07-13T15:49:11.627Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"239e60a0-6a0a-4c1f-90f3-f3779bc65ff4"}
{"level":"info","ts":"2026-07-13T15:49:11.627Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"239e60a0-6a0a-4c1f-90f3-f3779bc65ff4","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:11.627Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"239e60a0-6a0a-4c1f-90f3-f3779bc65ff4"}
{"level":"info","ts":"2026-07-13T15:49:11.627Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"239e60a0-6a0a-4c1f-90f3-f3779bc65ff4","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:11.627Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"239e60a0-6a0a-4c1f-90f3-f3779bc65ff4"}
{"level":"info","ts":"2026-07-13T15:49:11.627Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"239e60a0-6a0a-4c1f-90f3-f3779bc65ff4","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:11.627Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"239e60a0-6a0a-4c1f-90f3-f3779bc65ff4","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:11.627Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"239e60a0-6a0a-4c1f-90f3-f3779bc65ff4","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T15:49:11.627Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"239e60a0-6a0a-4c1f-90f3-f3779bc65ff4","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T15:49:11.627Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"239e60a0-6a0a-4c1f-90f3-f3779bc65ff4","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T15:49:11.627Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"239e60a0-6a0a-4c1f-90f3-f3779bc65ff4","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:11.627Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"239e60a0-6a0a-4c1f-90f3-f3779bc65ff4","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:11.627Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"239e60a0-6a0a-4c1f-90f3-f3779bc65ff4","enabled":true}
{"level":"error","ts":"2026-07-13T15:49:11.631Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"239e60a0-6a0a-4c1f-90f3-f3779bc65ff4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:11.631Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"239e60a0-6a0a-4c1f-90f3-f3779bc65ff4","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:49:11.631Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"239e60a0-6a0a-4c1f-90f3-f3779bc65ff4","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:11.631Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"239e60a0-6a0a-4c1f-90f3-f3779bc65ff4","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:11.631Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"239e60a0-6a0a-4c1f-90f3-f3779bc65ff4","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:11.631Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"239e60a0-6a0a-4c1f-90f3-f3779bc65ff4"}
{"level":"error","ts":"2026-07-13T15:49:11.631Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"239e60a0-6a0a-4c1f-90f3-f3779bc65ff4","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:12.912Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1a5362f8-68ee-434b-a120-36c48f6be60b"}
{"level":"info","ts":"2026-07-13T15:49:12.912Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1a5362f8-68ee-434b-a120-36c48f6be60b"}
{"level":"info","ts":"2026-07-13T15:49:12.912Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1a5362f8-68ee-434b-a120-36c48f6be60b","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:12.912Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1a5362f8-68ee-434b-a120-36c48f6be60b"}
{"level":"info","ts":"2026-07-13T15:49:12.912Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1a5362f8-68ee-434b-a120-36c48f6be60b","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:12.912Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1a5362f8-68ee-434b-a120-36c48f6be60b"}
{"level":"info","ts":"2026-07-13T15:49:12.912Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1a5362f8-68ee-434b-a120-36c48f6be60b","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:12.912Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1a5362f8-68ee-434b-a120-36c48f6be60b","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:12.912Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1a5362f8-68ee-434b-a120-36c48f6be60b","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T15:49:12.912Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1a5362f8-68ee-434b-a120-36c48f6be60b","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T15:49:12.912Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1a5362f8-68ee-434b-a120-36c48f6be60b","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T15:49:12.912Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1a5362f8-68ee-434b-a120-36c48f6be60b","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:12.912Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1a5362f8-68ee-434b-a120-36c48f6be60b","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-k9mkf.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:49:12.912Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1a5362f8-68ee-434b-a120-36c48f6be60b","enabled":true}
{"level":"error","ts":"2026-07-13T15:49:12.915Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1a5362f8-68ee-434b-a120-36c48f6be60b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:12.915Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1a5362f8-68ee-434b-a120-36c48f6be60b","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:49:12.915Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1a5362f8-68ee-434b-a120-36c48f6be60b","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:12.915Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1a5362f8-68ee-434b-a120-36c48f6be60b","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:12.915Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1a5362f8-68ee-434b-a120-36c48f6be60b","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:12.915Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1a5362f8-68ee-434b-a120-36c48f6be60b"}
{"level":"error","ts":"2026-07-13T15:49:12.915Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"1a5362f8-68ee-434b-a120-36c48f6be60b","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:13.711Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9a437ad1-f98e-43ad-9610-880252822190"}
{"level":"info","ts":"2026-07-13T15:49:13.712Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9a437ad1-f98e-43ad-9610-880252822190"}
{"level":"info","ts":"2026-07-13T15:49:13.712Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9a437ad1-f98e-43ad-9610-880252822190","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:13.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9a437ad1-f98e-43ad-9610-880252822190","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:13.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9a437ad1-f98e-43ad-9610-880252822190","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:13.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9a437ad1-f98e-43ad-9610-880252822190","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:13.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9a437ad1-f98e-43ad-9610-880252822190","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:13.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9a437ad1-f98e-43ad-9610-880252822190","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:13.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9a437ad1-f98e-43ad-9610-880252822190","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:13.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9a437ad1-f98e-43ad-9610-880252822190","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:13.712Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9a437ad1-f98e-43ad-9610-880252822190","podName":"raycluster-head-fdvcn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:13.712Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9a437ad1-f98e-43ad-9610-880252822190","podName":"raycluster-head-fdvcn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:13.712Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9a437ad1-f98e-43ad-9610-880252822190","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:13.712Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"9a437ad1-f98e-43ad-9610-880252822190","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:13.801Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37e825fc-e34a-4342-8606-181e6d5bb05d"}
{"level":"info","ts":"2026-07-13T15:49:13.801Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37e825fc-e34a-4342-8606-181e6d5bb05d"}
{"level":"info","ts":"2026-07-13T15:49:13.801Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37e825fc-e34a-4342-8606-181e6d5bb05d","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:13.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37e825fc-e34a-4342-8606-181e6d5bb05d","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:13.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37e825fc-e34a-4342-8606-181e6d5bb05d","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:13.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37e825fc-e34a-4342-8606-181e6d5bb05d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:13.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37e825fc-e34a-4342-8606-181e6d5bb05d","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:13.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37e825fc-e34a-4342-8606-181e6d5bb05d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:13.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37e825fc-e34a-4342-8606-181e6d5bb05d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:13.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37e825fc-e34a-4342-8606-181e6d5bb05d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:13.801Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37e825fc-e34a-4342-8606-181e6d5bb05d","podName":"raycluster-head-fdvcn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:13.801Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37e825fc-e34a-4342-8606-181e6d5bb05d","podName":"raycluster-head-fdvcn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:13.801Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37e825fc-e34a-4342-8606-181e6d5bb05d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:13.801Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"37e825fc-e34a-4342-8606-181e6d5bb05d","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:14.746Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e780d625-78d4-4ce2-b61e-6e5f07b38ce8"}
{"level":"info","ts":"2026-07-13T15:49:14.746Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e780d625-78d4-4ce2-b61e-6e5f07b38ce8"}
{"level":"info","ts":"2026-07-13T15:49:14.746Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e780d625-78d4-4ce2-b61e-6e5f07b38ce8","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:14.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e780d625-78d4-4ce2-b61e-6e5f07b38ce8","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:14.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e780d625-78d4-4ce2-b61e-6e5f07b38ce8","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:14.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e780d625-78d4-4ce2-b61e-6e5f07b38ce8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:14.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e780d625-78d4-4ce2-b61e-6e5f07b38ce8","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:14.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e780d625-78d4-4ce2-b61e-6e5f07b38ce8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:14.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e780d625-78d4-4ce2-b61e-6e5f07b38ce8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:14.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e780d625-78d4-4ce2-b61e-6e5f07b38ce8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:14.747Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e780d625-78d4-4ce2-b61e-6e5f07b38ce8","podName":"raycluster-head-fdvcn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:14.747Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e780d625-78d4-4ce2-b61e-6e5f07b38ce8","podName":"raycluster-head-fdvcn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:14.747Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e780d625-78d4-4ce2-b61e-6e5f07b38ce8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:14.747Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"e780d625-78d4-4ce2-b61e-6e5f07b38ce8","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:14.849Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"ee0d805a-02c6-4748-980b-829b5104b959","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:14.853Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"ee0d805a-02c6-4748-980b-829b5104b959","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:14.853Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"14fafbdc-bf53-4d49-b3df-11ef92c25934","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:14.858Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"14fafbdc-bf53-4d49-b3df-11ef92c25934","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:14.935Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"cceb767c-2b46-4238-9346-1d11c29b2907","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"}}
{"level":"info","ts":"2026-07-13T15:49:14.935Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"ebf5f88d-e1fb-4d47-b10a-c14a51e7a47d"}
{"level":"info","ts":"2026-07-13T15:49:14.935Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"ebf5f88d-e1fb-4d47-b10a-c14a51e7a47d"}
{"level":"info","ts":"2026-07-13T15:49:14.935Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"cceb767c-2b46-4238-9346-1d11c29b2907","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:14.936Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"cceb767c-2b46-4238-9346-1d11c29b2907","cluster":"raycluster-auth"}
{"level":"error","ts":"2026-07-13T15:49:14.938Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"ebf5f88d-e1fb-4d47-b10a-c14a51e7a47d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcileHeadService-fm","error":"services \"raycluster-auth-head-svc\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:14.938Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"ebf5f88d-e1fb-4d47-b10a-c14a51e7a47d","error":"services \"raycluster-auth-head-svc\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:49:14.938Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"ebf5f88d-e1fb-4d47-b10a-c14a51e7a47d","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:14.938Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"ebf5f88d-e1fb-4d47-b10a-c14a51e7a47d","serviceSelectors":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/identifier":"raycluster-auth-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:14.938Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"ebf5f88d-e1fb-4d47-b10a-c14a51e7a47d","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:14.938Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"ebf5f88d-e1fb-4d47-b10a-c14a51e7a47d","error":"unable to find head service. cluster name raycluster-auth, filter labels [test-ns-k9mkf map[ray.io/cluster:raycluster-auth ray.io/identifier:raycluster-auth-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-07-13T15:49:14.938Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"ebf5f88d-e1fb-4d47-b10a-c14a51e7a47d"}
{"level":"error","ts":"2026-07-13T15:49:14.938Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"ebf5f88d-e1fb-4d47-b10a-c14a51e7a47d","error":"services \"raycluster-auth-head-svc\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:49:14.949Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"cceb767c-2b46-4238-9346-1d11c29b2907","error":"failed to ensure OIDC resources: failed to ensure ConfigMap: failed to create or update OIDC ConfigMap: configmaps \"kube-rbac-proxy-config-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:14.949Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"cceb767c-2b46-4238-9346-1d11c29b2907"}
{"level":"error","ts":"2026-07-13T15:49:14.949Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"cceb767c-2b46-4238-9346-1d11c29b2907","error":"failed to ensure OIDC resources: failed to ensure ConfigMap: failed to create or update OIDC ConfigMap: configmaps \"kube-rbac-proxy-config-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:14.949Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"fecbc75c-8192-4a73-a198-b611f2cda44b","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:14.949Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"eee831de-7f51-4c43-91a3-b403bac72ba6","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:14.950Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"eee831de-7f51-4c43-91a3-b403bac72ba6","rayCluster":"raycluster-auth"}
{"level":"error","ts":"2026-07-13T15:49:14.952Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"eee831de-7f51-4c43-91a3-b403bac72ba6","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:49:14.952Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"eee831de-7f51-4c43-91a3-b403bac72ba6","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:14.952Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"eee831de-7f51-4c43-91a3-b403bac72ba6"}
{"level":"error","ts":"2026-07-13T15:49:14.952Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"eee831de-7f51-4c43-91a3-b403bac72ba6","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:14.954Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"fecbc75c-8192-4a73-a198-b611f2cda44b","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:14.955Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"c4074733-4fb3-4b47-b9b4-858290912cee","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"}}
{"level":"info","ts":"2026-07-13T15:49:14.955Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"c4074733-4fb3-4b47-b9b4-858290912cee","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:14.955Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"c4074733-4fb3-4b47-b9b4-858290912cee","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:14.957Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"45823341-addc-4ca1-9037-30fc3f4a14c9","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:14.957Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"45823341-addc-4ca1-9037-30fc3f4a14c9","rayCluster":"raycluster-auth"}
{"level":"error","ts":"2026-07-13T15:49:14.959Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"45823341-addc-4ca1-9037-30fc3f4a14c9","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:49:14.959Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"45823341-addc-4ca1-9037-30fc3f4a14c9","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:14.959Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"45823341-addc-4ca1-9037-30fc3f4a14c9"}
{"level":"error","ts":"2026-07-13T15:49:14.959Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"45823341-addc-4ca1-9037-30fc3f4a14c9","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:49:14.965Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"c4074733-4fb3-4b47-b9b4-858290912cee","error":"failed to ensure OIDC resources: failed to ensure ConfigMap: failed to create or update OIDC ConfigMap: configmaps \"kube-rbac-proxy-config-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:14.965Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"c4074733-4fb3-4b47-b9b4-858290912cee"}
{"level":"error","ts":"2026-07-13T15:49:14.965Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"c4074733-4fb3-4b47-b9b4-858290912cee","error":"failed to ensure OIDC resources: failed to ensure ConfigMap: failed to create or update OIDC ConfigMap: configmaps \"kube-rbac-proxy-config-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:14.969Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"090fd7d2-b1c7-45e9-a210-b8368c53883c","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:14.969Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"090fd7d2-b1c7-45e9-a210-b8368c53883c","rayCluster":"raycluster-auth"}
{"level":"error","ts":"2026-07-13T15:49:14.972Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"090fd7d2-b1c7-45e9-a210-b8368c53883c","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:49:14.972Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"090fd7d2-b1c7-45e9-a210-b8368c53883c","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:14.972Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"090fd7d2-b1c7-45e9-a210-b8368c53883c"}
{"level":"error","ts":"2026-07-13T15:49:14.972Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"090fd7d2-b1c7-45e9-a210-b8368c53883c","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:14.976Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e33100aa-87f1-45e7-9c61-a3c0be9a29c5","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"}}
{"level":"info","ts":"2026-07-13T15:49:14.976Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e33100aa-87f1-45e7-9c61-a3c0be9a29c5","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:14.976Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e33100aa-87f1-45e7-9c61-a3c0be9a29c5","cluster":"raycluster-auth"}
{"level":"error","ts":"2026-07-13T15:49:14.986Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e33100aa-87f1-45e7-9c61-a3c0be9a29c5","error":"failed to ensure OIDC resources: failed to ensure ConfigMap: failed to create or update OIDC ConfigMap: configmaps \"kube-rbac-proxy-config-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:14.986Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e33100aa-87f1-45e7-9c61-a3c0be9a29c5"}
{"level":"error","ts":"2026-07-13T15:49:14.986Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e33100aa-87f1-45e7-9c61-a3c0be9a29c5","error":"failed to ensure OIDC resources: failed to ensure ConfigMap: failed to create or update OIDC ConfigMap: configmaps \"kube-rbac-proxy-config-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:14.992Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"7f04284c-b208-4999-9eb8-8f4882b2b7d8","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:14.992Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"7f04284c-b208-4999-9eb8-8f4882b2b7d8","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.007Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e7623de1-ab95-487f-8f15-250f5f66c5a3","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"}}
{"level":"info","ts":"2026-07-13T15:49:15.007Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e7623de1-ab95-487f-8f15-250f5f66c5a3","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.007Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e7623de1-ab95-487f-8f15-250f5f66c5a3","cluster":"raycluster-auth"}
{"level":"error","ts":"2026-07-13T15:49:15.018Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e7623de1-ab95-487f-8f15-250f5f66c5a3","error":"failed to ensure OIDC resources: failed to ensure ConfigMap: failed to create or update OIDC ConfigMap: configmaps \"kube-rbac-proxy-config-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.018Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e7623de1-ab95-487f-8f15-250f5f66c5a3"}
{"level":"error","ts":"2026-07-13T15:49:15.018Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"e7623de1-ab95-487f-8f15-250f5f66c5a3","error":"failed to ensure OIDC resources: failed to ensure ConfigMap: failed to create or update OIDC ConfigMap: configmaps \"kube-rbac-proxy-config-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:49:14.995Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"7f04284c-b208-4999-9eb8-8f4882b2b7d8","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:49:15.021Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"7f04284c-b208-4999-9eb8-8f4882b2b7d8","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.021Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"7f04284c-b208-4999-9eb8-8f4882b2b7d8"}
{"level":"error","ts":"2026-07-13T15:49:15.021Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"7f04284c-b208-4999-9eb8-8f4882b2b7d8","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.058Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"1d336110-0d6a-420c-9a35-392a166ac1cf","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"}}
{"level":"info","ts":"2026-07-13T15:49:15.058Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"1d336110-0d6a-420c-9a35-392a166ac1cf","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.058Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"1d336110-0d6a-420c-9a35-392a166ac1cf","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.061Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a5ab3f26-b112-4e1b-9b02-eedba344e290","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.061Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a5ab3f26-b112-4e1b-9b02-eedba344e290","rayCluster":"raycluster-auth"}
{"level":"error","ts":"2026-07-13T15:49:15.063Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a5ab3f26-b112-4e1b-9b02-eedba344e290","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:49:15.063Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a5ab3f26-b112-4e1b-9b02-eedba344e290","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.063Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a5ab3f26-b112-4e1b-9b02-eedba344e290"}
{"level":"error","ts":"2026-07-13T15:49:15.063Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a5ab3f26-b112-4e1b-9b02-eedba344e290","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:49:15.069Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"1d336110-0d6a-420c-9a35-392a166ac1cf","error":"failed to ensure OIDC resources: failed to ensure ConfigMap: failed to create or update OIDC ConfigMap: configmaps \"kube-rbac-proxy-config-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.069Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"1d336110-0d6a-420c-9a35-392a166ac1cf"}
{"level":"error","ts":"2026-07-13T15:49:15.069Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"1d336110-0d6a-420c-9a35-392a166ac1cf","error":"failed to ensure OIDC resources: failed to ensure ConfigMap: failed to create or update OIDC ConfigMap: configmaps \"kube-rbac-proxy-config-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.143Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"46777062-9dfd-457b-bf7f-80bc917d9d5a","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.143Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"46777062-9dfd-457b-bf7f-80bc917d9d5a","rayCluster":"raycluster-auth"}
{"level":"error","ts":"2026-07-13T15:49:15.146Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"46777062-9dfd-457b-bf7f-80bc917d9d5a","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:49:15.146Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"46777062-9dfd-457b-bf7f-80bc917d9d5a","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.146Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"46777062-9dfd-457b-bf7f-80bc917d9d5a"}
{"level":"error","ts":"2026-07-13T15:49:15.146Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"46777062-9dfd-457b-bf7f-80bc917d9d5a","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.150Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"bea544d2-d5e1-41bf-a8bb-8721d7299ce9","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"}}
{"level":"info","ts":"2026-07-13T15:49:15.150Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"bea544d2-d5e1-41bf-a8bb-8721d7299ce9","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.150Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"bea544d2-d5e1-41bf-a8bb-8721d7299ce9","cluster":"raycluster-auth"}
{"level":"error","ts":"2026-07-13T15:49:15.230Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"bea544d2-d5e1-41bf-a8bb-8721d7299ce9","error":"failed to ensure OIDC resources: failed to ensure ConfigMap: failed to create or update OIDC ConfigMap: configmaps \"kube-rbac-proxy-config-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.231Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"bea544d2-d5e1-41bf-a8bb-8721d7299ce9"}
{"level":"error","ts":"2026-07-13T15:49:15.231Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"bea544d2-d5e1-41bf-a8bb-8721d7299ce9","error":"failed to ensure OIDC resources: failed to ensure ConfigMap: failed to create or update OIDC ConfigMap: configmaps \"kube-rbac-proxy-config-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.306Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"4b2cb411-60e9-4b91-a305-cb0a751735e8","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.306Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"4b2cb411-60e9-4b91-a305-cb0a751735e8","rayCluster":"raycluster-auth"}
{"level":"error","ts":"2026-07-13T15:49:15.309Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"4b2cb411-60e9-4b91-a305-cb0a751735e8","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:49:15.309Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"4b2cb411-60e9-4b91-a305-cb0a751735e8","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.309Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"4b2cb411-60e9-4b91-a305-cb0a751735e8"}
{"level":"error","ts":"2026-07-13T15:49:15.309Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"4b2cb411-60e9-4b91-a305-cb0a751735e8","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.391Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"fa6f96bc-1507-4354-9ce3-264bc8db9d2d","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"}}
{"level":"info","ts":"2026-07-13T15:49:15.391Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"fa6f96bc-1507-4354-9ce3-264bc8db9d2d","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.391Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"fa6f96bc-1507-4354-9ce3-264bc8db9d2d","cluster":"raycluster-auth"}
{"level":"error","ts":"2026-07-13T15:49:15.403Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"fa6f96bc-1507-4354-9ce3-264bc8db9d2d","error":"failed to ensure OIDC resources: failed to ensure ConfigMap: failed to create or update OIDC ConfigMap: configmaps \"kube-rbac-proxy-config-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.403Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"fa6f96bc-1507-4354-9ce3-264bc8db9d2d"}
{"level":"error","ts":"2026-07-13T15:49:15.403Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"fa6f96bc-1507-4354-9ce3-264bc8db9d2d","error":"failed to ensure OIDC resources: failed to ensure ConfigMap: failed to create or update OIDC ConfigMap: configmaps \"kube-rbac-proxy-config-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.441Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"26f12063-3bd5-4d43-b2a6-b1c25763ac9a","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"}}
{"level":"info","ts":"2026-07-13T15:49:15.441Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"26f12063-3bd5-4d43-b2a6-b1c25763ac9a","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.441Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"26f12063-3bd5-4d43-b2a6-b1c25763ac9a","cluster":"raycluster-auth"}
{"level":"error","ts":"2026-07-13T15:49:15.453Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"26f12063-3bd5-4d43-b2a6-b1c25763ac9a","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-auth-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.453Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"26f12063-3bd5-4d43-b2a6-b1c25763ac9a"}
{"level":"error","ts":"2026-07-13T15:49:15.453Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"26f12063-3bd5-4d43-b2a6-b1c25763ac9a","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-auth-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.454Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"7581c930-b916-4788-b3d5-ad3a63a33a89","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.454Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a9fafd4d-8785-47ce-a7ef-30ba5a96e69c","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.454Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a9fafd4d-8785-47ce-a7ef-30ba5a96e69c","rayCluster":"raycluster-auth"}
{"level":"error","ts":"2026-07-13T15:49:15.456Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a9fafd4d-8785-47ce-a7ef-30ba5a96e69c","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:49:15.456Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a9fafd4d-8785-47ce-a7ef-30ba5a96e69c","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.456Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a9fafd4d-8785-47ce-a7ef-30ba5a96e69c"}
{"level":"error","ts":"2026-07-13T15:49:15.456Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a9fafd4d-8785-47ce-a7ef-30ba5a96e69c","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.458Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"7581c930-b916-4788-b3d5-ad3a63a33a89","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.476Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"848f507a-9e94-4a58-86f7-1b5088277d0a"}
{"level":"info","ts":"2026-07-13T15:49:15.476Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"848f507a-9e94-4a58-86f7-1b5088277d0a"}
{"level":"error","ts":"2026-07-13T15:49:15.478Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"848f507a-9e94-4a58-86f7-1b5088277d0a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcileHeadService-fm","error":"services \"raycluster-auth-head-svc\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.478Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"848f507a-9e94-4a58-86f7-1b5088277d0a","error":"services \"raycluster-auth-head-svc\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:49:15.478Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"848f507a-9e94-4a58-86f7-1b5088277d0a","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:15.478Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"848f507a-9e94-4a58-86f7-1b5088277d0a","serviceSelectors":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/identifier":"raycluster-auth-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:15.478Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"848f507a-9e94-4a58-86f7-1b5088277d0a","filter labels":["test-ns-k9mkf",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:15.478Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"848f507a-9e94-4a58-86f7-1b5088277d0a","error":"unable to find head service. cluster name raycluster-auth, filter labels [test-ns-k9mkf map[ray.io/cluster:raycluster-auth ray.io/identifier:raycluster-auth-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-07-13T15:49:15.478Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"848f507a-9e94-4a58-86f7-1b5088277d0a"}
{"level":"error","ts":"2026-07-13T15:49:15.478Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"848f507a-9e94-4a58-86f7-1b5088277d0a","error":"services \"raycluster-auth-head-svc\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.577Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a3f2085b-974c-45b7-b6e6-eef9c3619af5","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"}}
{"level":"info","ts":"2026-07-13T15:49:15.577Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"2947aa44-4f46-466a-af68-685862ee0757","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.577Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"1d7a706c-a94f-4401-8653-609ab690e533"}
{"level":"info","ts":"2026-07-13T15:49:15.577Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a3f2085b-974c-45b7-b6e6-eef9c3619af5","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.577Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"2947aa44-4f46-466a-af68-685862ee0757","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.577Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"reconcileID":"b9100cd7-376d-4ac6-93f3-a228d95abf5e"}
{"level":"info","ts":"2026-07-13T15:49:15.577Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a3f2085b-974c-45b7-b6e6-eef9c3619af5","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.577Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a3f2085b-974c-45b7-b6e6-eef9c3619af5","namespace":"test-ns-k9mkf","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-13T15:49:15.577Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a3f2085b-974c-45b7-b6e6-eef9c3619af5","name":"kuberay-gateway-access"}
{"level":"error","ts":"2026-07-13T15:49:15.579Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"2947aa44-4f46-466a-af68-685862ee0757","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:49:15.579Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"2947aa44-4f46-466a-af68-685862ee0757","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.579Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"2947aa44-4f46-466a-af68-685862ee0757"}
{"level":"error","ts":"2026-07-13T15:49:15.579Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"2947aa44-4f46-466a-af68-685862ee0757","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-k9mkf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:15.582Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a3f2085b-974c-45b7-b6e6-eef9c3619af5","name":"test-ns-k9mkf-raycluster-auth","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-13T15:49:15.591Z","logger":"raycluster-resource","msg":"default","name":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.591Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster-auth","namespace":"test-ns-k9mkf"}
{"level":"info","ts":"2026-07-13T15:49:15.600Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"a3f2085b-974c-45b7-b6e6-eef9c3619af5","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.601Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"0cf23d3f-310d-4d17-9df9-0443a10a6455","namespace":"test-ns-k9mkf","clusterName":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.601Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"0cf23d3f-310d-4d17-9df9-0443a10a6455","namespace":"test-ns-k9mkf","clusterName":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.601Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"ad68a87f-1213-4d08-99f9-bac9f68194ac","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"}}
{"level":"info","ts":"2026-07-13T15:49:15.601Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"ad68a87f-1213-4d08-99f9-bac9f68194ac"}
{"level":"info","ts":"2026-07-13T15:49:15.601Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"ad68a87f-1213-4d08-99f9-bac9f68194ac","namespace":"test-ns-k9mkf","authClusters":0}
{"level":"info","ts":"2026-07-13T15:49:15.601Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"ad68a87f-1213-4d08-99f9-bac9f68194ac","namespace":"test-ns-k9mkf"}
{"level":"info","ts":"2026-07-13T15:49:15.601Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"ee75a027-8da9-4116-ae63-ae60c9eef19d"}
{"level":"info","ts":"2026-07-13T15:49:15.629Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"5e24305e-aa2b-490e-ba9c-ef73d522fba9","namespace":"test-ns-k9mkf","clusterName":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.629Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"5e24305e-aa2b-490e-ba9c-ef73d522fba9","namespace":"test-ns-k9mkf","clusterName":"raycluster-auth"}
{"level":"info","ts":"2026-07-13T15:49:15.724Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"06e0587c-3f9c-4cf7-b99d-52d24faec895","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"}}
{"level":"info","ts":"2026-07-13T15:49:15.724Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"06e0587c-3f9c-4cf7-b99d-52d24faec895"}
{"level":"info","ts":"2026-07-13T15:49:15.724Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"06e0587c-3f9c-4cf7-b99d-52d24faec895","namespace":"test-ns-k9mkf","authClusters":0}
{"level":"info","ts":"2026-07-13T15:49:15.724Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-k9mkf"},"namespace":"test-ns-k9mkf","name":"raycluster-auth","reconcileID":"06e0587c-3f9c-4cf7-b99d-52d24faec895","namespace":"test-ns-k9mkf"}
{"level":"info","ts":"2026-07-13T15:49:16.820Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"20373304-651c-4d17-a77e-ed9c85ed4c30"}
{"level":"info","ts":"2026-07-13T15:49:16.820Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"20373304-651c-4d17-a77e-ed9c85ed4c30"}
{"level":"info","ts":"2026-07-13T15:49:16.820Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"20373304-651c-4d17-a77e-ed9c85ed4c30","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:16.820Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"20373304-651c-4d17-a77e-ed9c85ed4c30","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:16.820Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"20373304-651c-4d17-a77e-ed9c85ed4c30","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:16.820Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"20373304-651c-4d17-a77e-ed9c85ed4c30","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:16.820Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"20373304-651c-4d17-a77e-ed9c85ed4c30","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:16.820Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"20373304-651c-4d17-a77e-ed9c85ed4c30","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:16.820Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"20373304-651c-4d17-a77e-ed9c85ed4c30","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:16.820Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"20373304-651c-4d17-a77e-ed9c85ed4c30","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:16.820Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"20373304-651c-4d17-a77e-ed9c85ed4c30","podName":"raycluster-head-fdvcn","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:49:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:49:16.820Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"20373304-651c-4d17-a77e-ed9c85ed4c30","podName":"raycluster-head-fdvcn","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:49:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:49:16.820Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"20373304-651c-4d17-a77e-ed9c85ed4c30","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:49:16.821Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"20373304-651c-4d17-a77e-ed9c85ed4c30","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:16.831Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"9fe74b8f-f0e5-4d44-9710-0a10e46c59bc","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:16.832Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"fda02eb8-ff3b-4a60-b933-438543c14ddb","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:16.832Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"9fe74b8f-f0e5-4d44-9710-0a10e46c59bc","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T15:49:16.832Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"9fe74b8f-f0e5-4d44-9710-0a10e46c59bc","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T15:49:16.832Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"9fe74b8f-f0e5-4d44-9710-0a10e46c59bc","rayCluster":"raycluster","podIPs":["10.133.0.18","10.134.0.25"]}
{"level":"info","ts":"2026-07-13T15:49:16.840Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"fda02eb8-ff3b-4a60-b933-438543c14ddb","name":"raycluster-head"}
{"level":"info","ts":"2026-07-13T15:49:16.848Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"fda02eb8-ff3b-4a60-b933-438543c14ddb","name":"raycluster-workers"}
{"level":"info","ts":"2026-07-13T15:49:16.848Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"fda02eb8-ff3b-4a60-b933-438543c14ddb","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:18.832Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db699956-4080-4612-9a37-5fbca1c63e4f"}
{"level":"info","ts":"2026-07-13T15:49:18.832Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db699956-4080-4612-9a37-5fbca1c63e4f"}
{"level":"info","ts":"2026-07-13T15:49:18.832Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db699956-4080-4612-9a37-5fbca1c63e4f","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:18.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db699956-4080-4612-9a37-5fbca1c63e4f","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:18.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db699956-4080-4612-9a37-5fbca1c63e4f","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:18.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db699956-4080-4612-9a37-5fbca1c63e4f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:18.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db699956-4080-4612-9a37-5fbca1c63e4f","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:18.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db699956-4080-4612-9a37-5fbca1c63e4f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:18.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db699956-4080-4612-9a37-5fbca1c63e4f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:18.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db699956-4080-4612-9a37-5fbca1c63e4f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:18.832Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db699956-4080-4612-9a37-5fbca1c63e4f","podName":"raycluster-head-fdvcn","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:49:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:49:18.832Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db699956-4080-4612-9a37-5fbca1c63e4f","podName":"raycluster-head-fdvcn","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:49:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:49:18.833Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db699956-4080-4612-9a37-5fbca1c63e4f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:18.833Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db699956-4080-4612-9a37-5fbca1c63e4f","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:23.747Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"52ed0afd-415f-4d1d-94fd-ca1d6e78a1c7"}
{"level":"info","ts":"2026-07-13T15:49:23.748Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"52ed0afd-415f-4d1d-94fd-ca1d6e78a1c7"}
{"level":"info","ts":"2026-07-13T15:49:23.748Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"52ed0afd-415f-4d1d-94fd-ca1d6e78a1c7","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:23.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"52ed0afd-415f-4d1d-94fd-ca1d6e78a1c7","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:23.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"52ed0afd-415f-4d1d-94fd-ca1d6e78a1c7","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:23.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"52ed0afd-415f-4d1d-94fd-ca1d6e78a1c7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:23.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"52ed0afd-415f-4d1d-94fd-ca1d6e78a1c7","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:23.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"52ed0afd-415f-4d1d-94fd-ca1d6e78a1c7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:23.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"52ed0afd-415f-4d1d-94fd-ca1d6e78a1c7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:23.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"52ed0afd-415f-4d1d-94fd-ca1d6e78a1c7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:23.748Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"52ed0afd-415f-4d1d-94fd-ca1d6e78a1c7","podName":"raycluster-small-group-worker-96vng","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:23.748Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"52ed0afd-415f-4d1d-94fd-ca1d6e78a1c7","podName":"raycluster-small-group-worker-96vng","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:23.748Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"52ed0afd-415f-4d1d-94fd-ca1d6e78a1c7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:23.748Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"52ed0afd-415f-4d1d-94fd-ca1d6e78a1c7","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:24.754Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"1fefb74d-42aa-431f-b24a-c6cc31744963"}
{"level":"info","ts":"2026-07-13T15:49:24.754Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"1fefb74d-42aa-431f-b24a-c6cc31744963"}
{"level":"info","ts":"2026-07-13T15:49:24.754Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"1fefb74d-42aa-431f-b24a-c6cc31744963","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:24.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"1fefb74d-42aa-431f-b24a-c6cc31744963","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:24.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"1fefb74d-42aa-431f-b24a-c6cc31744963","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:24.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"1fefb74d-42aa-431f-b24a-c6cc31744963","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:24.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"1fefb74d-42aa-431f-b24a-c6cc31744963","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:24.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"1fefb74d-42aa-431f-b24a-c6cc31744963","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:24.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"1fefb74d-42aa-431f-b24a-c6cc31744963","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:24.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"1fefb74d-42aa-431f-b24a-c6cc31744963","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:24.754Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"1fefb74d-42aa-431f-b24a-c6cc31744963","podName":"raycluster-head-fdvcn","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:49:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:49:24.754Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"1fefb74d-42aa-431f-b24a-c6cc31744963","podName":"raycluster-head-fdvcn","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:49:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:49:24.754Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"1fefb74d-42aa-431f-b24a-c6cc31744963","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:49:24.754Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"1fefb74d-42aa-431f-b24a-c6cc31744963","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:24.765Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"ad0b2640-512c-468d-92e8-8cb56e8d6116","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:24.765Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"c5ad7d17-f9cd-4d21-ab47-c148f27dc34f","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:24.766Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"c5ad7d17-f9cd-4d21-ab47-c148f27dc34f","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T15:49:24.766Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"c5ad7d17-f9cd-4d21-ab47-c148f27dc34f","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T15:49:24.766Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"c5ad7d17-f9cd-4d21-ab47-c148f27dc34f","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T15:49:24.776Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"ad0b2640-512c-468d-92e8-8cb56e8d6116","name":"raycluster-head"}
{"level":"info","ts":"2026-07-13T15:49:24.783Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"ad0b2640-512c-468d-92e8-8cb56e8d6116","name":"raycluster-workers"}
{"level":"info","ts":"2026-07-13T15:49:24.783Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"ad0b2640-512c-468d-92e8-8cb56e8d6116","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:26.765Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fb613def-b7f0-4ef1-8473-6d45aeea1c36"}
{"level":"info","ts":"2026-07-13T15:49:26.765Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fb613def-b7f0-4ef1-8473-6d45aeea1c36"}
{"level":"info","ts":"2026-07-13T15:49:26.765Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fb613def-b7f0-4ef1-8473-6d45aeea1c36","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:26.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fb613def-b7f0-4ef1-8473-6d45aeea1c36","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:26.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fb613def-b7f0-4ef1-8473-6d45aeea1c36","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:26.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fb613def-b7f0-4ef1-8473-6d45aeea1c36","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:26.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fb613def-b7f0-4ef1-8473-6d45aeea1c36","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:26.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fb613def-b7f0-4ef1-8473-6d45aeea1c36","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:26.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fb613def-b7f0-4ef1-8473-6d45aeea1c36","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:26.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fb613def-b7f0-4ef1-8473-6d45aeea1c36","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:26.766Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fb613def-b7f0-4ef1-8473-6d45aeea1c36","podName":"raycluster-head-fdvcn","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:49:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:49:26.766Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fb613def-b7f0-4ef1-8473-6d45aeea1c36","podName":"raycluster-head-fdvcn","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:49:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:49:26.766Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fb613def-b7f0-4ef1-8473-6d45aeea1c36","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:26.766Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fb613def-b7f0-4ef1-8473-6d45aeea1c36","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:27.848Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"40b891f1-f622-4c3d-a494-7ec3d48a45d1"}
{"level":"info","ts":"2026-07-13T15:49:27.848Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"40b891f1-f622-4c3d-a494-7ec3d48a45d1"}
{"level":"info","ts":"2026-07-13T15:49:27.848Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"40b891f1-f622-4c3d-a494-7ec3d48a45d1","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:27.848Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"40b891f1-f622-4c3d-a494-7ec3d48a45d1","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:27.848Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"40b891f1-f622-4c3d-a494-7ec3d48a45d1","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:27.848Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"40b891f1-f622-4c3d-a494-7ec3d48a45d1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:27.848Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"40b891f1-f622-4c3d-a494-7ec3d48a45d1","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:27.848Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"40b891f1-f622-4c3d-a494-7ec3d48a45d1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:27.848Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"40b891f1-f622-4c3d-a494-7ec3d48a45d1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:27.848Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"40b891f1-f622-4c3d-a494-7ec3d48a45d1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:27.848Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"40b891f1-f622-4c3d-a494-7ec3d48a45d1","podName":"raycluster-small-group-worker-96vng","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:49:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:49:27.848Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"40b891f1-f622-4c3d-a494-7ec3d48a45d1","podName":"raycluster-small-group-worker-96vng","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:49:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:49:27.848Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"40b891f1-f622-4c3d-a494-7ec3d48a45d1","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:49:27.848Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"40b891f1-f622-4c3d-a494-7ec3d48a45d1","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:27.859Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"deca7a14-167f-422b-b1d4-7958a5b859d7","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:27.859Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"458cacef-ab20-4998-80c6-4abf7771e237","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:27.859Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"deca7a14-167f-422b-b1d4-7958a5b859d7","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T15:49:27.859Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"deca7a14-167f-422b-b1d4-7958a5b859d7","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T15:49:27.859Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"deca7a14-167f-422b-b1d4-7958a5b859d7","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T15:49:27.867Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"458cacef-ab20-4998-80c6-4abf7771e237","name":"raycluster-head"}
{"level":"info","ts":"2026-07-13T15:49:27.877Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"458cacef-ab20-4998-80c6-4abf7771e237","name":"raycluster-workers"}
{"level":"info","ts":"2026-07-13T15:49:27.877Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"458cacef-ab20-4998-80c6-4abf7771e237","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:29.859Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d54a9970-f317-40bd-a8ba-a5d126c341d7"}
{"level":"info","ts":"2026-07-13T15:49:29.859Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d54a9970-f317-40bd-a8ba-a5d126c341d7"}
{"level":"info","ts":"2026-07-13T15:49:29.859Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d54a9970-f317-40bd-a8ba-a5d126c341d7","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:29.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d54a9970-f317-40bd-a8ba-a5d126c341d7","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:29.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d54a9970-f317-40bd-a8ba-a5d126c341d7","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:29.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d54a9970-f317-40bd-a8ba-a5d126c341d7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:29.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d54a9970-f317-40bd-a8ba-a5d126c341d7","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:29.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d54a9970-f317-40bd-a8ba-a5d126c341d7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:29.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d54a9970-f317-40bd-a8ba-a5d126c341d7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:29.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d54a9970-f317-40bd-a8ba-a5d126c341d7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:29.859Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d54a9970-f317-40bd-a8ba-a5d126c341d7","podName":"raycluster-small-group-worker-96vng","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:49:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:49:29.859Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d54a9970-f317-40bd-a8ba-a5d126c341d7","podName":"raycluster-small-group-worker-96vng","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:49:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:49:29.860Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d54a9970-f317-40bd-a8ba-a5d126c341d7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:29.860Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d54a9970-f317-40bd-a8ba-a5d126c341d7","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:34.932Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"edb1c44e-c716-4188-9ae2-e17262817c5f","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:34.933Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"edb1c44e-c716-4188-9ae2-e17262817c5f","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T15:49:34.933Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"edb1c44e-c716-4188-9ae2-e17262817c5f","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T15:49:34.933Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"edb1c44e-c716-4188-9ae2-e17262817c5f","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T15:49:35.780Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"13f0150f-337e-4b55-a0c8-4d88299c2075"}
{"level":"info","ts":"2026-07-13T15:49:35.780Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"13f0150f-337e-4b55-a0c8-4d88299c2075"}
{"level":"info","ts":"2026-07-13T15:49:35.780Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"13f0150f-337e-4b55-a0c8-4d88299c2075","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:35.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"13f0150f-337e-4b55-a0c8-4d88299c2075","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:35.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"13f0150f-337e-4b55-a0c8-4d88299c2075","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:35.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"13f0150f-337e-4b55-a0c8-4d88299c2075","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:35.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"13f0150f-337e-4b55-a0c8-4d88299c2075","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:35.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"13f0150f-337e-4b55-a0c8-4d88299c2075","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:35.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"13f0150f-337e-4b55-a0c8-4d88299c2075","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:35.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"13f0150f-337e-4b55-a0c8-4d88299c2075","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:35.780Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"13f0150f-337e-4b55-a0c8-4d88299c2075","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-13T15:49:35.780Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"13f0150f-337e-4b55-a0c8-4d88299c2075","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:35.792Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"e998830e-b535-4a39-975d-c89a402cad29","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:35.792Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"64661159-bac9-4ae4-96de-88b1d73f1316","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:35.792Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"e998830e-b535-4a39-975d-c89a402cad29","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T15:49:35.792Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"e998830e-b535-4a39-975d-c89a402cad29","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T15:49:35.792Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"e998830e-b535-4a39-975d-c89a402cad29","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T15:49:35.803Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"64661159-bac9-4ae4-96de-88b1d73f1316","name":"raycluster-head"}
{"level":"info","ts":"2026-07-13T15:49:35.811Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"64661159-bac9-4ae4-96de-88b1d73f1316","name":"raycluster-workers"}
{"level":"info","ts":"2026-07-13T15:49:35.811Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"64661159-bac9-4ae4-96de-88b1d73f1316","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:49:36.753Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"9d946f41-e95a-4c8e-9f2d-675f18dd3472","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:49:36.753Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"9d946f41-e95a-4c8e-9f2d-675f18dd3472","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T15:49:36.764Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"9d946f41-e95a-4c8e-9f2d-675f18dd3472"}
{"level":"info","ts":"2026-07-13T15:49:36.764Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"9d946f41-e95a-4c8e-9f2d-675f18dd3472","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-13T15:49:36.764Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"9d946f41-e95a-4c8e-9f2d-675f18dd3472","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dhwqc","rayClusterName":"counter-qn4mr","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:49:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:49:36.764Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"9d946f41-e95a-4c8e-9f2d-675f18dd3472","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T15:49:36.775Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"10f6614c-d7a1-4021-95f6-779e7163b405","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:49:36.775Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"10f6614c-d7a1-4021-95f6-779e7163b405"}
{"level":"info","ts":"2026-07-13T15:49:36.775Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"10f6614c-d7a1-4021-95f6-779e7163b405","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-13T15:49:36.775Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"10f6614c-d7a1-4021-95f6-779e7163b405","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-ptxj6","rayClusterName":"counter-pm7qc","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:49:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:49:36.775Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"10f6614c-d7a1-4021-95f6-779e7163b405","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T15:49:36.780Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"10f6614c-d7a1-4021-95f6-779e7163b405","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-13T15:49:36.780Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"10f6614c-d7a1-4021-95f6-779e7163b405"}
{"level":"error","ts":"2026-07-13T15:49:36.780Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"10f6614c-d7a1-4021-95f6-779e7163b405","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:36.780Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"6e5dd48c-ec35-4646-aa34-a39e2ebd7064","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:49:36.780Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"6e5dd48c-ec35-4646-aa34-a39e2ebd7064","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.780Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"6e5dd48c-ec35-4646-aa34-a39e2ebd7064","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:36.780Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"6e5dd48c-ec35-4646-aa34-a39e2ebd7064","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:36.784Z","logger":"raycluster-resource","msg":"default","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.784Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-qn4mr","namespace":"test-ns-5xvz7"}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"6e5dd48c-ec35-4646-aa34-a39e2ebd7064","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"6e5dd48c-ec35-4646-aa34-a39e2ebd7064"}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"6e5dd48c-ec35-4646-aa34-a39e2ebd7064","RayCluster":"counter-qn4mr","State":""}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"906a04b8-4ba3-4f9e-9a6e-d3e3d717c14e","namespacedName":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"9894e6e5-acda-4622-a2df-d6cfb6d8765f","rayCluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"9894e6e5-acda-4622-a2df-d6cfb6d8765f","rayCluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ffa8f9ca-f6bd-45c9-9eb6-e5bf1b601b72","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"0078705e-1f95-4561-8df4-8d233e2a5a2e","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e6d6e40e-7b2e-4c60-ba0d-82bf960a7a27"}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e6d6e40e-7b2e-4c60-ba0d-82bf960a7a27"}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"906a04b8-4ba3-4f9e-9a6e-d3e3d717c14e","mode":"IntegratedOAuth","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"906a04b8-4ba3-4f9e-9a6e-d3e3d717c14e","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"906a04b8-4ba3-4f9e-9a6e-d3e3d717c14e","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ffa8f9ca-f6bd-45c9-9eb6-e5bf1b601b72","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ffa8f9ca-f6bd-45c9-9eb6-e5bf1b601b72","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ffa8f9ca-f6bd-45c9-9eb6-e5bf1b601b72"}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ffa8f9ca-f6bd-45c9-9eb6-e5bf1b601b72","RayCluster":"counter-qn4mr","State":""}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"9819c9a3-9bdb-4088-87f0-019b36d6496b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"9819c9a3-9bdb-4088-87f0-019b36d6496b","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"9819c9a3-9bdb-4088-87f0-019b36d6496b","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"9819c9a3-9bdb-4088-87f0-019b36d6496b"}
{"level":"info","ts":"2026-07-13T15:49:36.793Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"9819c9a3-9bdb-4088-87f0-019b36d6496b","RayCluster":"counter-qn4mr","State":""}
{"level":"info","ts":"2026-07-13T15:49:36.798Z","logger":"raycluster-resource","msg":"default","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.798Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-qn4mr","namespace":"test-ns-5xvz7"}
{"level":"info","ts":"2026-07-13T15:49:36.801Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e6d6e40e-7b2e-4c60-ba0d-82bf960a7a27","name":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:36.801Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e6d6e40e-7b2e-4c60-ba0d-82bf960a7a27","error":"head certificate secret ray-head-secret-counter-qn4mr not found"}
{"level":"error","ts":"2026-07-13T15:49:36.801Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e6d6e40e-7b2e-4c60-ba0d-82bf960a7a27","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:36.804Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"9894e6e5-acda-4622-a2df-d6cfb6d8765f","rayCluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.821Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e6d6e40e-7b2e-4c60-ba0d-82bf960a7a27","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found"}
{"level":"info","ts":"2026-07-13T15:49:36.821Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"9894e6e5-acda-4622-a2df-d6cfb6d8765f","rayCluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.806Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"056aeeab-a317-4ca9-9790-3bf1574876c4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:49:36.821Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"056aeeab-a317-4ca9-9790-3bf1574876c4","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.821Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e6d6e40e-7b2e-4c60-ba0d-82bf960a7a27","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:36.821Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"056aeeab-a317-4ca9-9790-3bf1574876c4","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:36.821Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e6d6e40e-7b2e-4c60-ba0d-82bf960a7a27","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:36.821Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"0078705e-1f95-4561-8df4-8d233e2a5a2e","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.821Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e6d6e40e-7b2e-4c60-ba0d-82bf960a7a27","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:36.821Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e6d6e40e-7b2e-4c60-ba0d-82bf960a7a27","oldEndpoints":null,"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{},"newHead":{"serviceName":"counter-qn4mr-head-svc"}}
{"level":"info","ts":"2026-07-13T15:49:36.821Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"d98d8cfd-e940-47d9-8ee4-c12c241c3e88","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.821Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e6d6e40e-7b2e-4c60-ba0d-82bf960a7a27","name":"counter-qn4mr","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-qn4mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:36.821Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"056aeeab-a317-4ca9-9790-3bf1574876c4"}
{"level":"info","ts":"2026-07-13T15:49:36.821Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"056aeeab-a317-4ca9-9790-3bf1574876c4","RayCluster":"counter-qn4mr","State":""}
{"level":"info","ts":"2026-07-13T15:49:36.826Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"906a04b8-4ba3-4f9e-9a6e-d3e3d717c14e","name":"counter-qn4mr-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:36.829Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e6d6e40e-7b2e-4c60-ba0d-82bf960a7a27","name":"counter-qn4mr","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-qn4mr\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-5xvz7","name":"counter-qn4mr"}}
{"level":"info","ts":"2026-07-13T15:49:36.829Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e6d6e40e-7b2e-4c60-ba0d-82bf960a7a27"}
{"level":"error","ts":"2026-07-13T15:49:36.829Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e6d6e40e-7b2e-4c60-ba0d-82bf960a7a27","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:36.829Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2f2148c0-968d-4cf8-b820-fee982719225"}
{"level":"info","ts":"2026-07-13T15:49:36.829Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2f2148c0-968d-4cf8-b820-fee982719225"}
{"level":"info","ts":"2026-07-13T15:49:36.829Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2f2148c0-968d-4cf8-b820-fee982719225","1 head service found":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:36.829Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2f2148c0-968d-4cf8-b820-fee982719225","error":"head certificate secret ray-head-secret-counter-qn4mr not found"}
{"level":"error","ts":"2026-07-13T15:49:36.829Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2f2148c0-968d-4cf8-b820-fee982719225","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:36.829Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2f2148c0-968d-4cf8-b820-fee982719225","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found"}
{"level":"info","ts":"2026-07-13T15:49:36.829Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2f2148c0-968d-4cf8-b820-fee982719225","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:36.829Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2f2148c0-968d-4cf8-b820-fee982719225","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:36.829Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2f2148c0-968d-4cf8-b820-fee982719225","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:36.829Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2f2148c0-968d-4cf8-b820-fee982719225","oldEndpoints":null,"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{},"newHead":{"serviceName":"counter-qn4mr-head-svc"}}
{"level":"info","ts":"2026-07-13T15:49:36.829Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2f2148c0-968d-4cf8-b820-fee982719225","name":"counter-qn4mr","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-qn4mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:36.829Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"9894e6e5-acda-4622-a2df-d6cfb6d8765f","rayCluster":"counter-qn4mr","secret":"ca-secret"}
{"level":"info","ts":"2026-07-13T15:49:36.830Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"9894e6e5-acda-4622-a2df-d6cfb6d8765f","rayCluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.833Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"906a04b8-4ba3-4f9e-9a6e-d3e3d717c14e","name":"kuberay-gateway-access","namespace":"test-ns-5xvz7","operation":"created"}
{"level":"info","ts":"2026-07-13T15:49:36.834Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"d98d8cfd-e940-47d9-8ee4-c12c241c3e88","name":"counter-qn4mr-head"}
{"level":"info","ts":"2026-07-13T15:49:36.840Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2f2148c0-968d-4cf8-b820-fee982719225"}
{"level":"error","ts":"2026-07-13T15:49:36.840Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2f2148c0-968d-4cf8-b820-fee982719225","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:36.840Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"88b5a320-5cfb-48bf-8b5b-06803c0a6162"}
{"level":"info","ts":"2026-07-13T15:49:36.840Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"88b5a320-5cfb-48bf-8b5b-06803c0a6162"}
{"level":"info","ts":"2026-07-13T15:49:36.840Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"88b5a320-5cfb-48bf-8b5b-06803c0a6162","1 head service found":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:36.840Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"88b5a320-5cfb-48bf-8b5b-06803c0a6162","error":"head certificate secret ray-head-secret-counter-qn4mr not found"}
{"level":"error","ts":"2026-07-13T15:49:36.840Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"88b5a320-5cfb-48bf-8b5b-06803c0a6162","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:36.840Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"88b5a320-5cfb-48bf-8b5b-06803c0a6162","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found"}
{"level":"info","ts":"2026-07-13T15:49:36.840Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"88b5a320-5cfb-48bf-8b5b-06803c0a6162","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:36.840Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"88b5a320-5cfb-48bf-8b5b-06803c0a6162","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:36.840Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"88b5a320-5cfb-48bf-8b5b-06803c0a6162","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:36.840Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"88b5a320-5cfb-48bf-8b5b-06803c0a6162","oldEndpoints":null,"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{},"newHead":{"serviceName":"counter-qn4mr-head-svc"}}
{"level":"info","ts":"2026-07-13T15:49:36.840Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"88b5a320-5cfb-48bf-8b5b-06803c0a6162","name":"counter-qn4mr","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-qn4mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:36.840Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"17b2303d-e520-4fe0-909d-f6e148e9c867","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:49:36.840Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"17b2303d-e520-4fe0-909d-f6e148e9c867","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.840Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"17b2303d-e520-4fe0-909d-f6e148e9c867","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:36.840Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"17b2303d-e520-4fe0-909d-f6e148e9c867"}
{"level":"info","ts":"2026-07-13T15:49:36.840Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"17b2303d-e520-4fe0-909d-f6e148e9c867","RayCluster":"counter-qn4mr","State":""}
{"level":"info","ts":"2026-07-13T15:49:36.842Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"906a04b8-4ba3-4f9e-9a6e-d3e3d717c14e","name":"test-ns-5xvz7-counter-qn4mr","operation":"created"}
{"level":"info","ts":"2026-07-13T15:49:36.843Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"9894e6e5-acda-4622-a2df-d6cfb6d8765f","rayCluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.921Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"88b5a320-5cfb-48bf-8b5b-06803c0a6162","name":"counter-qn4mr","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-qn4mr\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-5xvz7","name":"counter-qn4mr"}}
{"level":"info","ts":"2026-07-13T15:49:36.921Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"88b5a320-5cfb-48bf-8b5b-06803c0a6162"}
{"level":"error","ts":"2026-07-13T15:49:36.921Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"88b5a320-5cfb-48bf-8b5b-06803c0a6162","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:36.921Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"d98d8cfd-e940-47d9-8ee4-c12c241c3e88","name":"counter-qn4mr-workers"}
{"level":"info","ts":"2026-07-13T15:49:36.921Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"d98d8cfd-e940-47d9-8ee4-c12c241c3e88","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.921Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"a5b807b2-3ca5-4017-b725-58ab99f46e74","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.921Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"9894e6e5-acda-4622-a2df-d6cfb6d8765f","rayCluster":"counter-qn4mr","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:49:36.921Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"16dffe9e-2ffb-4829-8633-7f52942e0a61"}
{"level":"info","ts":"2026-07-13T15:49:36.921Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"16dffe9e-2ffb-4829-8633-7f52942e0a61"}
{"level":"info","ts":"2026-07-13T15:49:36.921Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"16dffe9e-2ffb-4829-8633-7f52942e0a61","1 head service found":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:36.921Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"16dffe9e-2ffb-4829-8633-7f52942e0a61","error":"head certificate secret ray-head-secret-counter-qn4mr not found"}
{"level":"error","ts":"2026-07-13T15:49:36.921Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"16dffe9e-2ffb-4829-8633-7f52942e0a61","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:36.922Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"16dffe9e-2ffb-4829-8633-7f52942e0a61","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found"}
{"level":"info","ts":"2026-07-13T15:49:36.922Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"16dffe9e-2ffb-4829-8633-7f52942e0a61","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:36.922Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"16dffe9e-2ffb-4829-8633-7f52942e0a61","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:36.922Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"16dffe9e-2ffb-4829-8633-7f52942e0a61","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:36.922Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"16dffe9e-2ffb-4829-8633-7f52942e0a61"}
{"level":"info","ts":"2026-07-13T15:49:36.922Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"906a04b8-4ba3-4f9e-9a6e-d3e3d717c14e","name":"kube-rbac-proxy-config-counter-qn4mr","operation":"created"}
{"level":"error","ts":"2026-07-13T15:49:36.922Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"16dffe9e-2ffb-4829-8633-7f52942e0a61","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:49:36.938Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"906a04b8-4ba3-4f9e-9a6e-d3e3d717c14e","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-qn4mr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:36.938Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"906a04b8-4ba3-4f9e-9a6e-d3e3d717c14e"}
{"level":"error","ts":"2026-07-13T15:49:36.938Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"906a04b8-4ba3-4f9e-9a6e-d3e3d717c14e","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-qn4mr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:36.938Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"6b1a78f4-c200-4284-b921-2f73828b5655","namespacedName":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:36.938Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"6b1a78f4-c200-4284-b921-2f73828b5655","mode":"IntegratedOAuth","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.938Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"6b1a78f4-c200-4284-b921-2f73828b5655","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.938Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"9894e6e5-acda-4622-a2df-d6cfb6d8765f","rayCluster":"counter-qn4mr","podIPs":[]}
{"level":"error","ts":"2026-07-13T15:49:36.938Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"9894e6e5-acda-4622-a2df-d6cfb6d8765f","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-qn4mr\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:36.938Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"9894e6e5-acda-4622-a2df-d6cfb6d8765f"}
{"level":"error","ts":"2026-07-13T15:49:36.938Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"9894e6e5-acda-4622-a2df-d6cfb6d8765f","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-qn4mr\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:36.938Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"dc0b282e-c4c5-4cf5-8f8f-1b736ddc4e64","rayCluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.942Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"35d0d900-c77d-4130-9185-58940f8d3ff5"}
{"level":"info","ts":"2026-07-13T15:49:36.942Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"35d0d900-c77d-4130-9185-58940f8d3ff5"}
{"level":"info","ts":"2026-07-13T15:49:36.942Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"35d0d900-c77d-4130-9185-58940f8d3ff5","1 head service found":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:36.942Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"35d0d900-c77d-4130-9185-58940f8d3ff5","error":"head certificate secret ray-head-secret-counter-qn4mr not found"}
{"level":"error","ts":"2026-07-13T15:49:36.942Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"35d0d900-c77d-4130-9185-58940f8d3ff5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:36.942Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"35d0d900-c77d-4130-9185-58940f8d3ff5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found"}
{"level":"info","ts":"2026-07-13T15:49:36.942Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"35d0d900-c77d-4130-9185-58940f8d3ff5","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:36.942Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"35d0d900-c77d-4130-9185-58940f8d3ff5","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:36.942Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"35d0d900-c77d-4130-9185-58940f8d3ff5","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:36.942Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"35d0d900-c77d-4130-9185-58940f8d3ff5"}
{"level":"error","ts":"2026-07-13T15:49:36.942Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"35d0d900-c77d-4130-9185-58940f8d3ff5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:36.950Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"a5b807b2-3ca5-4017-b725-58ab99f46e74","name":"counter-qn4mr-head"}
{"level":"info","ts":"2026-07-13T15:49:36.966Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"6b1a78f4-c200-4284-b921-2f73828b5655","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.967Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"657d367e-13a6-4e88-9630-388308e112f3","namespacedName":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:36.967Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"657d367e-13a6-4e88-9630-388308e112f3","mode":"IntegratedOAuth","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.967Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"657d367e-13a6-4e88-9630-388308e112f3","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.967Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"f07b09ec-6fdc-4764-ac01-00afcd42dc22","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:49:36.967Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"f07b09ec-6fdc-4764-ac01-00afcd42dc22","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.967Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"f07b09ec-6fdc-4764-ac01-00afcd42dc22","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:36.967Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"f07b09ec-6fdc-4764-ac01-00afcd42dc22"}
{"level":"info","ts":"2026-07-13T15:49:36.967Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"f07b09ec-6fdc-4764-ac01-00afcd42dc22","RayCluster":"counter-qn4mr","State":""}
{"level":"error","ts":"2026-07-13T15:49:36.980Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"dc0b282e-c4c5-4cf5-8f8f-1b736ddc4e64","error":"certificates.cert-manager.io \"ray-worker-cert-counter-qn4mr\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:49:36.980Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"dc0b282e-c4c5-4cf5-8f8f-1b736ddc4e64","error":"certificates.cert-manager.io \"ray-worker-cert-counter-qn4mr\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:36.980Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"dc0b282e-c4c5-4cf5-8f8f-1b736ddc4e64"}
{"level":"error","ts":"2026-07-13T15:49:36.980Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"dc0b282e-c4c5-4cf5-8f8f-1b736ddc4e64","error":"certificates.cert-manager.io \"ray-worker-cert-counter-qn4mr\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:36.980Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"d572421a-22de-4d05-a508-66d896951bbb","rayCluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.980Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"d572421a-22de-4d05-a508-66d896951bbb"}
{"level":"error","ts":"2026-07-13T15:49:36.984Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"657d367e-13a6-4e88-9630-388308e112f3","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-qn4mr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:36.984Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"657d367e-13a6-4e88-9630-388308e112f3"}
{"level":"error","ts":"2026-07-13T15:49:36.984Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"657d367e-13a6-4e88-9630-388308e112f3","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-qn4mr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:36.986Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"a5b807b2-3ca5-4017-b725-58ab99f46e74","name":"counter-qn4mr-workers"}
{"level":"info","ts":"2026-07-13T15:49:36.986Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"a5b807b2-3ca5-4017-b725-58ab99f46e74","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.986Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"ed7c6598-3080-47ae-9351-85ce14fe7371","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.990Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"648124b3-3999-4921-8397-89f0b65bd103","namespacedName":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:36.990Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"648124b3-3999-4921-8397-89f0b65bd103","mode":"IntegratedOAuth","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:36.990Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"648124b3-3999-4921-8397-89f0b65bd103","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:37.021Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"7fce1ce7-0b3c-438c-8eac-40441f3ffe55","rayCluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:37.021Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"7fce1ce7-0b3c-438c-8eac-40441f3ffe55"}
{"level":"info","ts":"2026-07-13T15:49:37.021Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"648124b3-3999-4921-8397-89f0b65bd103","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:37.022Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"bf5bb8f4-fcef-4e1a-9bb9-a99b1dbcecbc"}
{"level":"info","ts":"2026-07-13T15:49:37.022Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"bf5bb8f4-fcef-4e1a-9bb9-a99b1dbcecbc"}
{"level":"info","ts":"2026-07-13T15:49:37.022Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"bf5bb8f4-fcef-4e1a-9bb9-a99b1dbcecbc","1 head service found":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:37.022Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"bf5bb8f4-fcef-4e1a-9bb9-a99b1dbcecbc","error":"head certificate secret ray-head-secret-counter-qn4mr not found"}
{"level":"error","ts":"2026-07-13T15:49:37.022Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"bf5bb8f4-fcef-4e1a-9bb9-a99b1dbcecbc","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:37.022Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"bf5bb8f4-fcef-4e1a-9bb9-a99b1dbcecbc","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found"}
{"level":"info","ts":"2026-07-13T15:49:37.022Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"bf5bb8f4-fcef-4e1a-9bb9-a99b1dbcecbc","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:37.022Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"bf5bb8f4-fcef-4e1a-9bb9-a99b1dbcecbc","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:37.022Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"bf5bb8f4-fcef-4e1a-9bb9-a99b1dbcecbc","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:37.022Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"bf5bb8f4-fcef-4e1a-9bb9-a99b1dbcecbc"}
{"level":"error","ts":"2026-07-13T15:49:37.023Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"bf5bb8f4-fcef-4e1a-9bb9-a99b1dbcecbc","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:37.030Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"ed7c6598-3080-47ae-9351-85ce14fe7371","name":"counter-qn4mr-head"}
{"level":"info","ts":"2026-07-13T15:49:37.046Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"ed7c6598-3080-47ae-9351-85ce14fe7371","name":"counter-qn4mr-workers"}
{"level":"info","ts":"2026-07-13T15:49:37.046Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"ed7c6598-3080-47ae-9351-85ce14fe7371","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:37.183Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"ac21694e-d388-495f-be6a-372c19008aca"}
{"level":"info","ts":"2026-07-13T15:49:37.183Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"ac21694e-d388-495f-be6a-372c19008aca"}
{"level":"info","ts":"2026-07-13T15:49:37.183Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"ac21694e-d388-495f-be6a-372c19008aca","1 head service found":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:37.183Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"ac21694e-d388-495f-be6a-372c19008aca","error":"head certificate secret ray-head-secret-counter-qn4mr not found"}
{"level":"error","ts":"2026-07-13T15:49:37.183Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"ac21694e-d388-495f-be6a-372c19008aca","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:37.183Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"ac21694e-d388-495f-be6a-372c19008aca","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found"}
{"level":"info","ts":"2026-07-13T15:49:37.183Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"ac21694e-d388-495f-be6a-372c19008aca","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:37.183Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"ac21694e-d388-495f-be6a-372c19008aca","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:37.183Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"ac21694e-d388-495f-be6a-372c19008aca","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:37.183Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"ac21694e-d388-495f-be6a-372c19008aca"}
{"level":"error","ts":"2026-07-13T15:49:37.183Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"ac21694e-d388-495f-be6a-372c19008aca","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:37.504Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"301f9e98-c604-4bdf-8b28-c366c7d92c56"}
{"level":"info","ts":"2026-07-13T15:49:37.504Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"301f9e98-c604-4bdf-8b28-c366c7d92c56"}
{"level":"info","ts":"2026-07-13T15:49:37.504Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"301f9e98-c604-4bdf-8b28-c366c7d92c56","1 head service found":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:37.504Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"301f9e98-c604-4bdf-8b28-c366c7d92c56","error":"head certificate secret ray-head-secret-counter-qn4mr not found"}
{"level":"error","ts":"2026-07-13T15:49:37.504Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"301f9e98-c604-4bdf-8b28-c366c7d92c56","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:37.504Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"301f9e98-c604-4bdf-8b28-c366c7d92c56","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found"}
{"level":"info","ts":"2026-07-13T15:49:37.504Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"301f9e98-c604-4bdf-8b28-c366c7d92c56","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:37.504Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"301f9e98-c604-4bdf-8b28-c366c7d92c56","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:37.504Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"301f9e98-c604-4bdf-8b28-c366c7d92c56","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:37.504Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"301f9e98-c604-4bdf-8b28-c366c7d92c56"}
{"level":"error","ts":"2026-07-13T15:49:37.504Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"301f9e98-c604-4bdf-8b28-c366c7d92c56","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:37.792Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ef1c105d-ea7c-4ef8-bcdb-3decbddad39a"}
{"level":"info","ts":"2026-07-13T15:49:37.792Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ef1c105d-ea7c-4ef8-bcdb-3decbddad39a"}
{"level":"info","ts":"2026-07-13T15:49:37.792Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ef1c105d-ea7c-4ef8-bcdb-3decbddad39a","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:37.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ef1c105d-ea7c-4ef8-bcdb-3decbddad39a","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:37.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ef1c105d-ea7c-4ef8-bcdb-3decbddad39a","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:37.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ef1c105d-ea7c-4ef8-bcdb-3decbddad39a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:49:37.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ef1c105d-ea7c-4ef8-bcdb-3decbddad39a","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:37.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ef1c105d-ea7c-4ef8-bcdb-3decbddad39a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:37.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ef1c105d-ea7c-4ef8-bcdb-3decbddad39a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:49:37.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ef1c105d-ea7c-4ef8-bcdb-3decbddad39a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:49:37.792Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ef1c105d-ea7c-4ef8-bcdb-3decbddad39a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:37.792Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ef1c105d-ea7c-4ef8-bcdb-3decbddad39a","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:38.145Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"71689ae6-00a5-44fb-9a44-4071bc30d8e5"}
{"level":"info","ts":"2026-07-13T15:49:38.145Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"71689ae6-00a5-44fb-9a44-4071bc30d8e5"}
{"level":"info","ts":"2026-07-13T15:49:38.145Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"71689ae6-00a5-44fb-9a44-4071bc30d8e5","1 head service found":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:38.145Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"71689ae6-00a5-44fb-9a44-4071bc30d8e5","error":"head certificate secret ray-head-secret-counter-qn4mr not found"}
{"level":"error","ts":"2026-07-13T15:49:38.145Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"71689ae6-00a5-44fb-9a44-4071bc30d8e5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:38.145Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"71689ae6-00a5-44fb-9a44-4071bc30d8e5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found"}
{"level":"info","ts":"2026-07-13T15:49:38.145Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"71689ae6-00a5-44fb-9a44-4071bc30d8e5","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:38.145Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"71689ae6-00a5-44fb-9a44-4071bc30d8e5","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:38.145Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"71689ae6-00a5-44fb-9a44-4071bc30d8e5","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"counter-qn4mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:49:38.146Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"71689ae6-00a5-44fb-9a44-4071bc30d8e5"}
{"level":"error","ts":"2026-07-13T15:49:38.146Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"71689ae6-00a5-44fb-9a44-4071bc30d8e5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-qn4mr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:39.427Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f9986412-30f5-4602-84b7-337ba16843ec"}
{"level":"info","ts":"2026-07-13T15:49:39.427Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f9986412-30f5-4602-84b7-337ba16843ec"}
{"level":"info","ts":"2026-07-13T15:49:39.427Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f9986412-30f5-4602-84b7-337ba16843ec","1 head service found":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:39.427Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f9986412-30f5-4602-84b7-337ba16843ec"}
{"level":"info","ts":"2026-07-13T15:49:39.427Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f9986412-30f5-4602-84b7-337ba16843ec","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:39.427Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f9986412-30f5-4602-84b7-337ba16843ec"}
{"level":"info","ts":"2026-07-13T15:49:39.427Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f9986412-30f5-4602-84b7-337ba16843ec","mode":"IntegratedOAuth","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:39.427Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f9986412-30f5-4602-84b7-337ba16843ec","cluster":"counter-qn4mr","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:49:39.427Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f9986412-30f5-4602-84b7-337ba16843ec","cluster":"counter-qn4mr","authType":"IntegratedOAuth","serviceAccount":"counter-qn4mr-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T15:49:39.427Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f9986412-30f5-4602-84b7-337ba16843ec","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-qn4mr","ray.io/group":"headgroup","ray.io/identifier":"counter-qn4mr-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T15:49:39.427Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f9986412-30f5-4602-84b7-337ba16843ec","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"counter-qn4mr-head-svc.test-ns-5xvz7.svc.cluster.local","num-cpus":"4","num-gpus":"4","resources":"'{\"R1\": 4}'"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T15:49:39.427Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f9986412-30f5-4602-84b7-337ba16843ec","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-qn4mr-head-svc.test-ns-5xvz7.svc.cluster.local  --num-cpus=4  --num-gpus=4  --resources='{\"R1\": 4}' "}
{"level":"info","ts":"2026-07-13T15:49:39.427Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f9986412-30f5-4602-84b7-337ba16843ec","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-qn4mr-head-svc.test-ns-5xvz7.svc.cluster.local  --num-cpus=4  --num-gpus=4  --resources='{\"R1\": 4}' "}
{"level":"info","ts":"2026-07-13T15:49:39.427Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f9986412-30f5-4602-84b7-337ba16843ec","enabled":true}
{"level":"info","ts":"2026-07-13T15:49:39.444Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f9986412-30f5-4602-84b7-337ba16843ec","name":"counter-qn4mr-head-dqjzn"}
{"level":"info","ts":"2026-07-13T15:49:39.444Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f9986412-30f5-4602-84b7-337ba16843ec","podName":"counter-qn4mr-head-dqjzn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:39.444Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f9986412-30f5-4602-84b7-337ba16843ec","podName":"counter-qn4mr-head-dqjzn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:39.444Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f9986412-30f5-4602-84b7-337ba16843ec","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-qn4mr-head-svc"},"newHead":{"podName":"counter-qn4mr-head-dqjzn","serviceName":"counter-qn4mr-head-svc"}}
{"level":"info","ts":"2026-07-13T15:49:39.444Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f9986412-30f5-4602-84b7-337ba16843ec","name":"counter-qn4mr","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-qn4mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-qn4mr-head-dqjzn","serviceName":"counter-qn4mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:39.455Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"cf0dc1c3-bce1-4b7f-b331-b4a05e5ca0c4"}
{"level":"info","ts":"2026-07-13T15:49:39.455Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"cf0dc1c3-bce1-4b7f-b331-b4a05e5ca0c4"}
{"level":"info","ts":"2026-07-13T15:49:39.455Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"cf0dc1c3-bce1-4b7f-b331-b4a05e5ca0c4","1 head service found":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:39.455Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"cf0dc1c3-bce1-4b7f-b331-b4a05e5ca0c4","Found 1 head Pod":"counter-qn4mr-head-dqjzn","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:39.455Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"cf0dc1c3-bce1-4b7f-b331-b4a05e5ca0c4","head Pod":"counter-qn4mr-head-dqjzn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-qn4mr-head-dqjzn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:39.455Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"cf0dc1c3-bce1-4b7f-b331-b4a05e5ca0c4","podName":"counter-qn4mr-head-dqjzn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:39.455Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"cf0dc1c3-bce1-4b7f-b331-b4a05e5ca0c4","podName":"counter-qn4mr-head-dqjzn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:39.455Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"cf0dc1c3-bce1-4b7f-b331-b4a05e5ca0c4","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-qn4mr-head-svc"},"newHead":{"podName":"counter-qn4mr-head-dqjzn","serviceName":"counter-qn4mr-head-svc"}}
{"level":"info","ts":"2026-07-13T15:49:39.455Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"cf0dc1c3-bce1-4b7f-b331-b4a05e5ca0c4","name":"counter-qn4mr","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-qn4mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-qn4mr-head-dqjzn","serviceName":"counter-qn4mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:39.455Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"6427221c-c188-4e2f-bead-40e106bee33b","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:39.455Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"c040a3e2-7aa3-458e-8b22-303c4e732dfe","rayCluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:39.455Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"9b648b15-2eb3-42fc-a0cb-97bb2d0f36e5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:49:39.455Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"9b648b15-2eb3-42fc-a0cb-97bb2d0f36e5","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:39.455Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"9b648b15-2eb3-42fc-a0cb-97bb2d0f36e5","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:39.455Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"c040a3e2-7aa3-458e-8b22-303c4e732dfe","rayCluster":"counter-qn4mr","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:49:39.455Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"9b648b15-2eb3-42fc-a0cb-97bb2d0f36e5"}
{"level":"info","ts":"2026-07-13T15:49:39.455Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"9b648b15-2eb3-42fc-a0cb-97bb2d0f36e5","RayCluster":"counter-qn4mr","State":""}
{"level":"info","ts":"2026-07-13T15:49:39.465Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"cf0dc1c3-bce1-4b7f-b331-b4a05e5ca0c4","name":"counter-qn4mr","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-qn4mr\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-5xvz7","name":"counter-qn4mr"}}
{"level":"info","ts":"2026-07-13T15:49:39.465Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"cf0dc1c3-bce1-4b7f-b331-b4a05e5ca0c4"}
{"level":"error","ts":"2026-07-13T15:49:39.465Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"cf0dc1c3-bce1-4b7f-b331-b4a05e5ca0c4","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-qn4mr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:49:39.466Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2ed423c0-f705-489c-9db2-40a507dbe492"}
{"level":"info","ts":"2026-07-13T15:49:39.466Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2ed423c0-f705-489c-9db2-40a507dbe492"}
{"level":"info","ts":"2026-07-13T15:49:39.466Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2ed423c0-f705-489c-9db2-40a507dbe492","1 head service found":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:39.466Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2ed423c0-f705-489c-9db2-40a507dbe492","Found 1 head Pod":"counter-qn4mr-head-dqjzn","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:39.466Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2ed423c0-f705-489c-9db2-40a507dbe492","head Pod":"counter-qn4mr-head-dqjzn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-qn4mr-head-dqjzn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:39.466Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2ed423c0-f705-489c-9db2-40a507dbe492","podName":"counter-qn4mr-head-dqjzn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:39.466Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2ed423c0-f705-489c-9db2-40a507dbe492","podName":"counter-qn4mr-head-dqjzn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:39.466Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2ed423c0-f705-489c-9db2-40a507dbe492","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:39.466Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"2ed423c0-f705-489c-9db2-40a507dbe492","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:39.471Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"7829942a-4c1f-4985-815f-c8703ed7d602"}
{"level":"info","ts":"2026-07-13T15:49:39.471Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"7829942a-4c1f-4985-815f-c8703ed7d602"}
{"level":"info","ts":"2026-07-13T15:49:39.471Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"7829942a-4c1f-4985-815f-c8703ed7d602","1 head service found":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:39.471Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"7829942a-4c1f-4985-815f-c8703ed7d602","Found 1 head Pod":"counter-qn4mr-head-dqjzn","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:39.471Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"7829942a-4c1f-4985-815f-c8703ed7d602","head Pod":"counter-qn4mr-head-dqjzn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-qn4mr-head-dqjzn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:39.471Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"7829942a-4c1f-4985-815f-c8703ed7d602","podName":"counter-qn4mr-head-dqjzn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:39.471Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"6427221c-c188-4e2f-bead-40e106bee33b","name":"counter-qn4mr-head"}
{"level":"info","ts":"2026-07-13T15:49:39.471Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"7829942a-4c1f-4985-815f-c8703ed7d602","podName":"counter-qn4mr-head-dqjzn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:39.472Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"7829942a-4c1f-4985-815f-c8703ed7d602","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:39.472Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"7829942a-4c1f-4985-815f-c8703ed7d602","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:39.523Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"c91d5611-0a80-4218-a62d-1bc0e0119ccd"}
{"level":"info","ts":"2026-07-13T15:49:39.523Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"c91d5611-0a80-4218-a62d-1bc0e0119ccd"}
{"level":"info","ts":"2026-07-13T15:49:39.523Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"c91d5611-0a80-4218-a62d-1bc0e0119ccd","1 head service found":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:39.523Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"c91d5611-0a80-4218-a62d-1bc0e0119ccd","Found 1 head Pod":"counter-qn4mr-head-dqjzn","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:39.523Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"c91d5611-0a80-4218-a62d-1bc0e0119ccd","head Pod":"counter-qn4mr-head-dqjzn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-qn4mr-head-dqjzn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:39.523Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"c91d5611-0a80-4218-a62d-1bc0e0119ccd","podName":"counter-qn4mr-head-dqjzn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:39.523Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"c91d5611-0a80-4218-a62d-1bc0e0119ccd","podName":"counter-qn4mr-head-dqjzn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:39.523Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"c91d5611-0a80-4218-a62d-1bc0e0119ccd","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:49:39.523Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"c91d5611-0a80-4218-a62d-1bc0e0119ccd","name":"counter-qn4mr","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-qn4mr-head-dqjzn","serviceName":"counter-qn4mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-qn4mr-head-dqjzn","serviceName":"counter-qn4mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:39.621Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"6427221c-c188-4e2f-bead-40e106bee33b","name":"counter-qn4mr-workers"}
{"level":"info","ts":"2026-07-13T15:49:39.621Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"6427221c-c188-4e2f-bead-40e106bee33b","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:39.622Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"43a96ec3-d2cc-4b57-b22c-18bccac10dcc","rayCluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:39.622Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"2b385f89-b878-4b1c-b1c5-93293c786dd7","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:39.622Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"1730e42e-f207-4f5e-ae66-b5f2dcd61216","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:49:39.622Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"1730e42e-f207-4f5e-ae66-b5f2dcd61216","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:39.622Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"1730e42e-f207-4f5e-ae66-b5f2dcd61216","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:39.622Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"43a96ec3-d2cc-4b57-b22c-18bccac10dcc","rayCluster":"counter-qn4mr","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:49:39.622Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"1730e42e-f207-4f5e-ae66-b5f2dcd61216"}
{"level":"info","ts":"2026-07-13T15:49:39.622Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"1730e42e-f207-4f5e-ae66-b5f2dcd61216","RayCluster":"counter-qn4mr","State":""}
{"level":"info","ts":"2026-07-13T15:49:39.726Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"2b385f89-b878-4b1c-b1c5-93293c786dd7","name":"counter-qn4mr-head"}
{"level":"info","ts":"2026-07-13T15:49:39.734Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"2b385f89-b878-4b1c-b1c5-93293c786dd7","name":"counter-qn4mr-workers"}
{"level":"info","ts":"2026-07-13T15:49:39.734Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"2b385f89-b878-4b1c-b1c5-93293c786dd7","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:39.794Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"69dc1752-585e-4148-ba7f-7a5a2ebf8064","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:49:39.794Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"69dc1752-585e-4148-ba7f-7a5a2ebf8064","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:39.794Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"69dc1752-585e-4148-ba7f-7a5a2ebf8064","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:39.794Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"69dc1752-585e-4148-ba7f-7a5a2ebf8064"}
{"level":"info","ts":"2026-07-13T15:49:39.794Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"69dc1752-585e-4148-ba7f-7a5a2ebf8064","RayCluster":"counter-qn4mr","State":""}
{"level":"info","ts":"2026-07-13T15:49:39.908Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"6d9011db-2bb7-443a-9f02-699e777497e3"}
{"level":"info","ts":"2026-07-13T15:49:39.908Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"6d9011db-2bb7-443a-9f02-699e777497e3"}
{"level":"info","ts":"2026-07-13T15:49:39.908Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"6d9011db-2bb7-443a-9f02-699e777497e3","1 head service found":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:39.908Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"6d9011db-2bb7-443a-9f02-699e777497e3","Found 1 head Pod":"counter-qn4mr-head-dqjzn","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:39.908Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"6d9011db-2bb7-443a-9f02-699e777497e3","head Pod":"counter-qn4mr-head-dqjzn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-qn4mr-head-dqjzn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:39.908Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"6d9011db-2bb7-443a-9f02-699e777497e3","podName":"counter-qn4mr-head-dqjzn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:39.908Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"6d9011db-2bb7-443a-9f02-699e777497e3","podName":"counter-qn4mr-head-dqjzn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:39.908Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"6d9011db-2bb7-443a-9f02-699e777497e3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:39.908Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"6d9011db-2bb7-443a-9f02-699e777497e3","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:40.815Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"ec42f3b8-dee9-49bb-85e3-f180d07089af"}
{"level":"info","ts":"2026-07-13T15:49:40.815Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"ec42f3b8-dee9-49bb-85e3-f180d07089af"}
{"level":"info","ts":"2026-07-13T15:49:40.815Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"ec42f3b8-dee9-49bb-85e3-f180d07089af","1 head service found":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:40.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"ec42f3b8-dee9-49bb-85e3-f180d07089af","Found 1 head Pod":"counter-qn4mr-head-dqjzn","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:40.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"ec42f3b8-dee9-49bb-85e3-f180d07089af","head Pod":"counter-qn4mr-head-dqjzn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-qn4mr-head-dqjzn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:40.815Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"ec42f3b8-dee9-49bb-85e3-f180d07089af","podName":"counter-qn4mr-head-dqjzn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:40.815Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"ec42f3b8-dee9-49bb-85e3-f180d07089af","podName":"counter-qn4mr-head-dqjzn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:40.816Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"ec42f3b8-dee9-49bb-85e3-f180d07089af","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"counter-qn4mr-head-dqjzn","serviceName":"counter-qn4mr-head-svc"},"newHead":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"counter-qn4mr-head-dqjzn","serviceName":"counter-qn4mr-head-svc"}}
{"level":"info","ts":"2026-07-13T15:49:40.816Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"ec42f3b8-dee9-49bb-85e3-f180d07089af","name":"counter-qn4mr","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-qn4mr-head-dqjzn","serviceName":"counter-qn4mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"counter-qn4mr-head-dqjzn","serviceName":"counter-qn4mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:40.827Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"70b48953-ff35-4f85-a444-609e8b68997a","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:40.827Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"cb116d29-e04c-4a78-85ba-d4b6be2eb800","rayCluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:40.827Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"cb116d29-e04c-4a78-85ba-d4b6be2eb800","pod":"counter-qn4mr-head-dqjzn","ip":"10.133.0.19"}
{"level":"info","ts":"2026-07-13T15:49:40.827Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"671c9c86-f905-45eb-8ed4-0879089d66e1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:49:40.827Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"671c9c86-f905-45eb-8ed4-0879089d66e1","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:40.827Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"671c9c86-f905-45eb-8ed4-0879089d66e1","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:40.827Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"671c9c86-f905-45eb-8ed4-0879089d66e1"}
{"level":"info","ts":"2026-07-13T15:49:40.827Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"671c9c86-f905-45eb-8ed4-0879089d66e1","RayCluster":"counter-qn4mr","State":""}
{"level":"info","ts":"2026-07-13T15:49:40.837Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"70b48953-ff35-4f85-a444-609e8b68997a","name":"counter-qn4mr-head"}
{"level":"info","ts":"2026-07-13T15:49:40.838Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"cb116d29-e04c-4a78-85ba-d4b6be2eb800","certificate":"ray-head-cert-counter-qn4mr","dnsNames":["counter-qn4mr-head-svc","localhost","counter-qn4mr-head-svc.test-ns-5xvz7.svc","counter-qn4mr-head-svc.test-ns-5xvz7.svc.cluster.local"],"ipAddresses":["10.133.0.19","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:49:40.848Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"70b48953-ff35-4f85-a444-609e8b68997a","name":"counter-qn4mr-workers"}
{"level":"info","ts":"2026-07-13T15:49:40.848Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"70b48953-ff35-4f85-a444-609e8b68997a","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:40.848Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"cb116d29-e04c-4a78-85ba-d4b6be2eb800","certificate":"ray-worker-cert-counter-qn4mr","dnsNames":["counter-qn4mr-worker-svc","localhost","counter-qn4mr-worker-svc.test-ns-5xvz7.svc","counter-qn4mr-worker-svc.test-ns-5xvz7.svc.cluster.local","*.counter-qn4mr-worker-svc.test-ns-5xvz7.svc","*.counter-qn4mr-worker-svc.test-ns-5xvz7.svc.cluster.local","*-worker-*.test-ns-5xvz7.svc","*-worker-*.test-ns-5xvz7.svc.cluster.local"],"ipAddresses":["10.133.0.19","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:49:40.848Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"cb116d29-e04c-4a78-85ba-d4b6be2eb800","rayCluster":"counter-qn4mr","podIPs":["10.133.0.19"]}
{"level":"info","ts":"2026-07-13T15:49:41.622Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e26f44cb-00cf-4c63-9c4a-1399227a7b57"}
{"level":"info","ts":"2026-07-13T15:49:41.622Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e26f44cb-00cf-4c63-9c4a-1399227a7b57"}
{"level":"info","ts":"2026-07-13T15:49:41.622Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e26f44cb-00cf-4c63-9c4a-1399227a7b57","1 head service found":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:41.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e26f44cb-00cf-4c63-9c4a-1399227a7b57","Found 1 head Pod":"counter-qn4mr-head-dqjzn","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:41.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e26f44cb-00cf-4c63-9c4a-1399227a7b57","head Pod":"counter-qn4mr-head-dqjzn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-qn4mr-head-dqjzn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:41.622Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e26f44cb-00cf-4c63-9c4a-1399227a7b57","podName":"counter-qn4mr-head-dqjzn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:41.622Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e26f44cb-00cf-4c63-9c4a-1399227a7b57","podName":"counter-qn4mr-head-dqjzn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:41.623Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e26f44cb-00cf-4c63-9c4a-1399227a7b57","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:41.623Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"e26f44cb-00cf-4c63-9c4a-1399227a7b57","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:42.794Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"6a9925c7-4439-4333-93d7-54b9cfc739a2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:49:42.794Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"6a9925c7-4439-4333-93d7-54b9cfc739a2","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:42.794Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"6a9925c7-4439-4333-93d7-54b9cfc739a2","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:42.794Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"6a9925c7-4439-4333-93d7-54b9cfc739a2"}
{"level":"info","ts":"2026-07-13T15:49:42.794Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"6a9925c7-4439-4333-93d7-54b9cfc739a2","RayCluster":"counter-qn4mr","State":""}
{"level":"info","ts":"2026-07-13T15:49:45.795Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ca83d32a-8c82-4d21-bf50-810c6301d9cf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:49:45.795Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ca83d32a-8c82-4d21-bf50-810c6301d9cf","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:45.795Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ca83d32a-8c82-4d21-bf50-810c6301d9cf","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:45.795Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ca83d32a-8c82-4d21-bf50-810c6301d9cf"}
{"level":"info","ts":"2026-07-13T15:49:45.795Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ca83d32a-8c82-4d21-bf50-810c6301d9cf","RayCluster":"counter-qn4mr","State":""}
{"level":"info","ts":"2026-07-13T15:49:45.836Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f3a4ebb1-a8ca-43ef-b743-05f8d97cf4a4"}
{"level":"info","ts":"2026-07-13T15:49:45.836Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f3a4ebb1-a8ca-43ef-b743-05f8d97cf4a4"}
{"level":"info","ts":"2026-07-13T15:49:45.836Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f3a4ebb1-a8ca-43ef-b743-05f8d97cf4a4","1 head service found":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:45.836Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f3a4ebb1-a8ca-43ef-b743-05f8d97cf4a4","Found 1 head Pod":"counter-qn4mr-head-dqjzn","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:45.836Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f3a4ebb1-a8ca-43ef-b743-05f8d97cf4a4","head Pod":"counter-qn4mr-head-dqjzn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-qn4mr-head-dqjzn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:45.836Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f3a4ebb1-a8ca-43ef-b743-05f8d97cf4a4","podName":"counter-qn4mr-head-dqjzn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:45.836Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f3a4ebb1-a8ca-43ef-b743-05f8d97cf4a4","podName":"counter-qn4mr-head-dqjzn","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:49:45.836Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f3a4ebb1-a8ca-43ef-b743-05f8d97cf4a4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:45.836Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"f3a4ebb1-a8ca-43ef-b743-05f8d97cf4a4","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:46.843Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"dc13041c-abe4-4b7b-adbb-2f38934d2256"}
{"level":"info","ts":"2026-07-13T15:49:46.843Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"dc13041c-abe4-4b7b-adbb-2f38934d2256"}
{"level":"info","ts":"2026-07-13T15:49:46.843Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"dc13041c-abe4-4b7b-adbb-2f38934d2256","1 head service found":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:46.844Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"dc13041c-abe4-4b7b-adbb-2f38934d2256","Found 1 head Pod":"counter-qn4mr-head-dqjzn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:46.844Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"dc13041c-abe4-4b7b-adbb-2f38934d2256","head Pod":"counter-qn4mr-head-dqjzn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-qn4mr-head-dqjzn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:46.844Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"dc13041c-abe4-4b7b-adbb-2f38934d2256","podName":"counter-qn4mr-head-dqjzn","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:49:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:49:46.844Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"dc13041c-abe4-4b7b-adbb-2f38934d2256","podName":"counter-qn4mr-head-dqjzn","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:49:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:49:46.844Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"dc13041c-abe4-4b7b-adbb-2f38934d2256","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:49:46.844Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"dc13041c-abe4-4b7b-adbb-2f38934d2256","name":"counter-qn4mr","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"counter-qn4mr-head-dqjzn","serviceName":"counter-qn4mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"counter-qn4mr-head-dqjzn","serviceName":"counter-qn4mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:46.857Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"247efd91-911b-4dd5-b0e1-cd5f04393264","rayCluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:46.857Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"247efd91-911b-4dd5-b0e1-cd5f04393264","pod":"counter-qn4mr-head-dqjzn","ip":"10.133.0.19"}
{"level":"info","ts":"2026-07-13T15:49:46.857Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"36a20dee-1352-4f46-8839-c4141fe0bcab","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:46.857Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"247efd91-911b-4dd5-b0e1-cd5f04393264","rayCluster":"counter-qn4mr","podIPs":["10.133.0.19"]}
{"level":"info","ts":"2026-07-13T15:49:46.857Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"0306df0f-1c69-4e3d-a3f1-04b0a780190f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:49:46.857Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"0306df0f-1c69-4e3d-a3f1-04b0a780190f","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:46.857Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"0306df0f-1c69-4e3d-a3f1-04b0a780190f","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:46.858Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"0306df0f-1c69-4e3d-a3f1-04b0a780190f"}
{"level":"info","ts":"2026-07-13T15:49:46.858Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"0306df0f-1c69-4e3d-a3f1-04b0a780190f","RayCluster":"counter-qn4mr","State":""}
{"level":"info","ts":"2026-07-13T15:49:46.868Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"36a20dee-1352-4f46-8839-c4141fe0bcab","name":"counter-qn4mr-head"}
{"level":"info","ts":"2026-07-13T15:49:46.879Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"36a20dee-1352-4f46-8839-c4141fe0bcab","name":"counter-qn4mr-workers"}
{"level":"info","ts":"2026-07-13T15:49:46.879Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"36a20dee-1352-4f46-8839-c4141fe0bcab","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:48.796Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"d419a8eb-f3b3-4270-9965-084b375443c4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:49:48.796Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"d419a8eb-f3b3-4270-9965-084b375443c4","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:48.796Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"d419a8eb-f3b3-4270-9965-084b375443c4","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:48.796Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"d419a8eb-f3b3-4270-9965-084b375443c4"}
{"level":"info","ts":"2026-07-13T15:49:48.796Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"d419a8eb-f3b3-4270-9965-084b375443c4","RayCluster":"counter-qn4mr","State":""}
{"level":"info","ts":"2026-07-13T15:49:48.858Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"53bd7ae3-3822-4fbc-87db-e448763452ff"}
{"level":"info","ts":"2026-07-13T15:49:48.858Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"53bd7ae3-3822-4fbc-87db-e448763452ff"}
{"level":"info","ts":"2026-07-13T15:49:48.858Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"53bd7ae3-3822-4fbc-87db-e448763452ff","1 head service found":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:48.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"53bd7ae3-3822-4fbc-87db-e448763452ff","Found 1 head Pod":"counter-qn4mr-head-dqjzn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:48.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"53bd7ae3-3822-4fbc-87db-e448763452ff","head Pod":"counter-qn4mr-head-dqjzn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-qn4mr-head-dqjzn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:48.858Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"53bd7ae3-3822-4fbc-87db-e448763452ff","podName":"counter-qn4mr-head-dqjzn","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:49:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:49:48.858Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"53bd7ae3-3822-4fbc-87db-e448763452ff","podName":"counter-qn4mr-head-dqjzn","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:49:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:49:48.858Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"53bd7ae3-3822-4fbc-87db-e448763452ff","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:48.858Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"53bd7ae3-3822-4fbc-87db-e448763452ff","seconds":300}
{"level":"info","ts":"2026-07-13T15:49:51.797Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"98b47c6c-bb0e-48a3-8411-fb7c24da14ab","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:49:51.797Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"98b47c6c-bb0e-48a3-8411-fb7c24da14ab","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:51.797Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"98b47c6c-bb0e-48a3-8411-fb7c24da14ab","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:51.797Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"98b47c6c-bb0e-48a3-8411-fb7c24da14ab"}
{"level":"info","ts":"2026-07-13T15:49:51.797Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"98b47c6c-bb0e-48a3-8411-fb7c24da14ab","RayCluster":"counter-qn4mr","State":""}
{"level":"info","ts":"2026-07-13T15:49:54.798Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"d8090b16-7d5e-4c4b-9943-1e02193153f7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:49:54.798Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"d8090b16-7d5e-4c4b-9943-1e02193153f7","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:54.798Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"d8090b16-7d5e-4c4b-9943-1e02193153f7","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:54.798Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"d8090b16-7d5e-4c4b-9943-1e02193153f7"}
{"level":"info","ts":"2026-07-13T15:49:54.798Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"d8090b16-7d5e-4c4b-9943-1e02193153f7","RayCluster":"counter-qn4mr","State":""}
{"level":"info","ts":"2026-07-13T15:49:57.799Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"c709eb72-af61-40a4-8fc0-f7cbb36fc6c9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:49:57.799Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"c709eb72-af61-40a4-8fc0-f7cbb36fc6c9","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:57.799Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"c709eb72-af61-40a4-8fc0-f7cbb36fc6c9","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:57.799Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"c709eb72-af61-40a4-8fc0-f7cbb36fc6c9"}
{"level":"info","ts":"2026-07-13T15:49:57.799Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"c709eb72-af61-40a4-8fc0-f7cbb36fc6c9","RayCluster":"counter-qn4mr","State":""}
{"level":"info","ts":"2026-07-13T15:49:57.870Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"c81e7fae-c659-4f7a-9eec-1cf253570ca5"}
{"level":"info","ts":"2026-07-13T15:49:57.870Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"c81e7fae-c659-4f7a-9eec-1cf253570ca5"}
{"level":"info","ts":"2026-07-13T15:49:57.870Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"c81e7fae-c659-4f7a-9eec-1cf253570ca5","1 head service found":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:57.870Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"c81e7fae-c659-4f7a-9eec-1cf253570ca5","Found 1 head Pod":"counter-qn4mr-head-dqjzn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:57.870Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"c81e7fae-c659-4f7a-9eec-1cf253570ca5","head Pod":"counter-qn4mr-head-dqjzn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-qn4mr-head-dqjzn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:57.870Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"c81e7fae-c659-4f7a-9eec-1cf253570ca5","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-13T15:49:57.870Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"c81e7fae-c659-4f7a-9eec-1cf253570ca5","name":"counter-qn4mr","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"counter-qn4mr-head-dqjzn","serviceName":"counter-qn4mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:57Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"counter-qn4mr-head-dqjzn","serviceName":"counter-qn4mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:49:57.881Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"b86a9641-eb89-4eac-8002-ecbd43b14d32","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:57.881Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"d2bc0ce0-2df0-46c4-b507-08a321b5062d","rayCluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:57.881Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"f5bfd9cc-7598-4bc3-a62e-4ed2a4cb47d8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:49:57.881Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"f5bfd9cc-7598-4bc3-a62e-4ed2a4cb47d8","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:57.881Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"f5bfd9cc-7598-4bc3-a62e-4ed2a4cb47d8","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:57.881Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"d2bc0ce0-2df0-46c4-b507-08a321b5062d","pod":"counter-qn4mr-head-dqjzn","ip":"10.133.0.19"}
{"level":"info","ts":"2026-07-13T15:49:57.881Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"d2bc0ce0-2df0-46c4-b507-08a321b5062d","rayCluster":"counter-qn4mr","podIPs":["10.133.0.19"]}
{"level":"info","ts":"2026-07-13T15:49:57.881Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"f5bfd9cc-7598-4bc3-a62e-4ed2a4cb47d8"}
{"level":"info","ts":"2026-07-13T15:49:57.881Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"f5bfd9cc-7598-4bc3-a62e-4ed2a4cb47d8","head service name":"counter-qn4mr-head-svc","namespace":"test-ns-5xvz7"}
{"level":"info","ts":"2026-07-13T15:49:57.882Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"f5bfd9cc-7598-4bc3-a62e-4ed2a4cb47d8","head service URL":"counter-qn4mr-head-svc.test-ns-5xvz7.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-13T15:49:57.882Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"f5bfd9cc-7598-4bc3-a62e-4ed2a4cb47d8","SubmissionMode":"HTTPMode","RayCluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:57.882Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"f5bfd9cc-7598-4bc3-a62e-4ed2a4cb47d8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dhwqc","rayClusterName":"counter-qn4mr","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:49:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dhwqc","rayClusterName":"counter-qn4mr","dashboardURL":"counter-qn4mr-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T15:49:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:49:57.882Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"f5bfd9cc-7598-4bc3-a62e-4ed2a4cb47d8","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T15:49:57.890Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"b86a9641-eb89-4eac-8002-ecbd43b14d32","name":"counter-qn4mr-head"}
{"level":"info","ts":"2026-07-13T15:49:57.891Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"a329ca01-71b5-4d8e-a364-fa4070dc5b6f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:49:57.891Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"a329ca01-71b5-4d8e-a364-fa4070dc5b6f","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:57.891Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"a329ca01-71b5-4d8e-a364-fa4070dc5b6f","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:49:57.891Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"a329ca01-71b5-4d8e-a364-fa4070dc5b6f"}
{"level":"info","ts":"2026-07-13T15:49:57.898Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"a329ca01-71b5-4d8e-a364-fa4070dc5b6f","JobId":"counter-dhwqc"}
{"level":"info","ts":"2026-07-13T15:49:57.898Z","logger":"controllers.RayJob","msg":"Submit a ray job","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"a329ca01-71b5-4d8e-a364-fa4070dc5b6f","rayJob":"counter","jobInfo":"{\"runtime_env\":{\"env_vars\":{\"counter_name\":\"test_counter\"}},\"entrypoint_resources\":{\"R1\":2},\"entrypoint\":\"python /home/ray/jobs/counter.py\",\"submission_id\":\"counter-dhwqc\",\"entrypoint_num_cpus\":2,\"entrypoint_num_gpus\":2}"}
{"level":"info","ts":"2026-07-13T15:49:57.900Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"b86a9641-eb89-4eac-8002-ecbd43b14d32","name":"counter-qn4mr-workers"}
{"level":"info","ts":"2026-07-13T15:49:57.900Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"b86a9641-eb89-4eac-8002-ecbd43b14d32","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:49:59.882Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"04e37e7c-be95-48db-9c74-cfcac0d68d50"}
{"level":"info","ts":"2026-07-13T15:49:59.882Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"04e37e7c-be95-48db-9c74-cfcac0d68d50"}
{"level":"info","ts":"2026-07-13T15:49:59.882Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"04e37e7c-be95-48db-9c74-cfcac0d68d50","1 head service found":"counter-qn4mr-head-svc"}
{"level":"info","ts":"2026-07-13T15:49:59.882Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"04e37e7c-be95-48db-9c74-cfcac0d68d50","Found 1 head Pod":"counter-qn4mr-head-dqjzn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:49:59.882Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"04e37e7c-be95-48db-9c74-cfcac0d68d50","head Pod":"counter-qn4mr-head-dqjzn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-qn4mr-head-dqjzn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:49:59.882Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"04e37e7c-be95-48db-9c74-cfcac0d68d50","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:49:59.882Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"04e37e7c-be95-48db-9c74-cfcac0d68d50","seconds":300}
{"level":"info","ts":"2026-07-13T15:50:00.800Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"bf1dcc22-4eff-4aa6-9e61-ad24f89e8a8a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:00.800Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"bf1dcc22-4eff-4aa6-9e61-ad24f89e8a8a","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:50:00.800Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"bf1dcc22-4eff-4aa6-9e61-ad24f89e8a8a","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:00.800Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"bf1dcc22-4eff-4aa6-9e61-ad24f89e8a8a"}
{"level":"info","ts":"2026-07-13T15:50:00.803Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"bf1dcc22-4eff-4aa6-9e61-ad24f89e8a8a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dhwqc","rayClusterName":"counter-qn4mr","dashboardURL":"counter-qn4mr-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T15:49:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:49:58Z"},"jobId":"counter-dhwqc","rayClusterName":"counter-qn4mr","dashboardURL":"counter-qn4mr-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T15:49:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:57Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"counter-qn4mr-head-dqjzn","serviceName":"counter-qn4mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:50:00.803Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"bf1dcc22-4eff-4aa6-9e61-ad24f89e8a8a","old JobStatus":"","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T15:50:00.815Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"bf78131e-96a7-4552-843e-aa29eddd918d","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:00.815Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"bf78131e-96a7-4552-843e-aa29eddd918d","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:50:00.815Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"bf78131e-96a7-4552-843e-aa29eddd918d","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:00.815Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"bf78131e-96a7-4552-843e-aa29eddd918d"}
{"level":"info","ts":"2026-07-13T15:50:00.817Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"bf78131e-96a7-4552-843e-aa29eddd918d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:49:58Z"},"jobId":"counter-dhwqc","rayClusterName":"counter-qn4mr","dashboardURL":"counter-qn4mr-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T15:49:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:57Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"counter-qn4mr-head-dqjzn","serviceName":"counter-qn4mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:49:58Z"},"jobId":"counter-dhwqc","rayClusterName":"counter-qn4mr","dashboardURL":"counter-qn4mr-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T15:49:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:57Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"counter-qn4mr-head-dqjzn","serviceName":"counter-qn4mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:50:03.814Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"23cbb524-0ce2-4518-a21c-3755216f75c7","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:03.814Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"23cbb524-0ce2-4518-a21c-3755216f75c7","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:50:03.815Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"23cbb524-0ce2-4518-a21c-3755216f75c7","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:03.815Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"23cbb524-0ce2-4518-a21c-3755216f75c7"}
{"level":"info","ts":"2026-07-13T15:50:03.818Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"23cbb524-0ce2-4518-a21c-3755216f75c7","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:49:58Z"},"jobId":"counter-dhwqc","rayClusterName":"counter-qn4mr","dashboardURL":"counter-qn4mr-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T15:49:36Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:57Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"counter-qn4mr-head-dqjzn","serviceName":"counter-qn4mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:49:58Z","endTime":"2026-07-13T15:50:03Z"},"jobId":"counter-dhwqc","rayClusterName":"counter-qn4mr","dashboardURL":"counter-qn4mr-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-07-13T15:49:36Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:57Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"counter-qn4mr-head-dqjzn","serviceName":"counter-qn4mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:36Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:36Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:50:03.818Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"23cbb524-0ce2-4518-a21c-3755216f75c7","old JobStatus":"RUNNING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-07-13T15:50:03.830Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ce2db15b-144e-4030-830f-1b75f2d4e5a5","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:03.830Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ce2db15b-144e-4030-830f-1b75f2d4e5a5","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 15:50:03 +0000 UTC","Now":"2026-07-13T15:50:03.830Z","ShutdownTime":"2026-07-13T15:50:03.000Z"}
{"level":"info","ts":"2026-07-13T15:50:03.830Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ce2db15b-144e-4030-830f-1b75f2d4e5a5","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T15:50:03.839Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ce2db15b-144e-4030-830f-1b75f2d4e5a5","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:03.839Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ce2db15b-144e-4030-830f-1b75f2d4e5a5","isClusterDeleted":false}
{"level":"info","ts":"2026-07-13T15:50:03.839Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ce2db15b-144e-4030-830f-1b75f2d4e5a5","RayCluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:50:03.839Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"a415e41c-961a-4f52-a5c6-8eba24ad5e40","namespacedName":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:03.839Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"572a1710-340c-4865-8bbc-6655f59f0b9d","rayCluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:50:03.839Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"reconcileID":"b0ca747c-1ac7-4a43-814c-e1946ae661a6"}
{"level":"info","ts":"2026-07-13T15:50:03.839Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"a415e41c-961a-4f52-a5c6-8eba24ad5e40","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:50:03.839Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"a415e41c-961a-4f52-a5c6-8eba24ad5e40","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:50:03.839Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"572a1710-340c-4865-8bbc-6655f59f0b9d","pod":"counter-qn4mr-head-dqjzn","ip":"10.133.0.19"}
{"level":"info","ts":"2026-07-13T15:50:03.839Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"98ef190c-a8d0-4e75-bb41-5fbfc5d9653c"}
{"level":"info","ts":"2026-07-13T15:50:03.839Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"a415e41c-961a-4f52-a5c6-8eba24ad5e40","namespace":"test-ns-5xvz7","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-13T15:50:03.840Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"572a1710-340c-4865-8bbc-6655f59f0b9d","rayCluster":"counter-qn4mr","podIPs":["10.133.0.19"]}
{"level":"info","ts":"2026-07-13T15:50:03.840Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ab573a7d-484c-46c6-8121-03babe9a7c16","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:03.840Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ab573a7d-484c-46c6-8121-03babe9a7c16","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 15:50:03 +0000 UTC","Now":"2026-07-13T15:50:03.840Z","ShutdownTime":"2026-07-13T15:50:03.000Z"}
{"level":"info","ts":"2026-07-13T15:50:03.840Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ab573a7d-484c-46c6-8121-03babe9a7c16","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T15:50:03.840Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ab573a7d-484c-46c6-8121-03babe9a7c16","RayCluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:50:03.840Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ab573a7d-484c-46c6-8121-03babe9a7c16","isClusterDeleted":false}
{"level":"info","ts":"2026-07-13T15:50:03.840Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ab573a7d-484c-46c6-8121-03babe9a7c16","RayCluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:50:03.844Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"a415e41c-961a-4f52-a5c6-8eba24ad5e40","name":"kuberay-gateway-access","namespace":"test-ns-5xvz7"}
{"level":"info","ts":"2026-07-13T15:50:03.849Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"a415e41c-961a-4f52-a5c6-8eba24ad5e40","name":"test-ns-5xvz7-counter-qn4mr","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-13T15:50:03.855Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"a415e41c-961a-4f52-a5c6-8eba24ad5e40","configMap":"kube-rbac-proxy-config-counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:50:03.859Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"a415e41c-961a-4f52-a5c6-8eba24ad5e40","serviceAccount":"counter-qn4mr-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:50:03.867Z","logger":"raycluster-resource","msg":"default","name":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:50:03.867Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-qn4mr","namespace":"test-ns-5xvz7"}
{"level":"info","ts":"2026-07-13T15:50:03.875Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"a415e41c-961a-4f52-a5c6-8eba24ad5e40","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:50:03.875Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"f01a782d-b9f2-4cc8-b85f-e07746654669","namespacedName":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:03.875Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"f01a782d-b9f2-4cc8-b85f-e07746654669","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:50:03.875Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"f01a782d-b9f2-4cc8-b85f-e07746654669","cluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:50:03.875Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"f01a782d-b9f2-4cc8-b85f-e07746654669","namespace":"test-ns-5xvz7","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-13T15:50:03.875Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"f01a782d-b9f2-4cc8-b85f-e07746654669","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-13T15:50:03.876Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"a0b5faa4-16da-4819-8b28-f5c89a73ec6e"}
{"level":"info","ts":"2026-07-13T15:50:03.876Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"35131f5b-90ec-4dec-9eb8-9565acf61987","namespace":"test-ns-5xvz7","clusterName":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:50:03.876Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"7b50c062-e513-4bac-9de8-f77631e3d2cb","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:03.876Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"7b50c062-e513-4bac-9de8-f77631e3d2cb","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 15:50:03 +0000 UTC","Now":"2026-07-13T15:50:03.876Z","ShutdownTime":"2026-07-13T15:50:03.000Z"}
{"level":"info","ts":"2026-07-13T15:50:03.876Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"7b50c062-e513-4bac-9de8-f77631e3d2cb","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T15:50:03.876Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"7b50c062-e513-4bac-9de8-f77631e3d2cb","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:03.876Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"7b50c062-e513-4bac-9de8-f77631e3d2cb","isClusterDeleted":true}
{"level":"info","ts":"2026-07-13T15:50:03.876Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"7b50c062-e513-4bac-9de8-f77631e3d2cb","RayCluster":"counter-qn4mr"}
{"level":"error","ts":"2026-07-13T15:50:03.878Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"f01a782d-b9f2-4cc8-b85f-e07746654669","error":"rayclusters.ray.io \"counter-qn4mr\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:03.879Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"f01a782d-b9f2-4cc8-b85f-e07746654669"}
{"level":"error","ts":"2026-07-13T15:50:03.879Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"f01a782d-b9f2-4cc8-b85f-e07746654669","error":"rayclusters.ray.io \"counter-qn4mr\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:03.879Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"4e8639ed-2d87-4011-9622-405802573f57","namespacedName":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:03.879Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"4e8639ed-2d87-4011-9622-405802573f57"}
{"level":"info","ts":"2026-07-13T15:50:03.879Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"4e8639ed-2d87-4011-9622-405802573f57","namespace":"test-ns-5xvz7","authClusters":0}
{"level":"info","ts":"2026-07-13T15:50:03.879Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"4e8639ed-2d87-4011-9622-405802573f57","namespace":"test-ns-5xvz7"}
{"level":"info","ts":"2026-07-13T15:50:03.883Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"73b1905c-cca6-466e-a48c-409db3231b03"}
{"level":"info","ts":"2026-07-13T15:50:03.884Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"e2415b26-5974-4863-9de0-c0a2bb21c41e","namespacedName":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:03.884Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"e2415b26-5974-4863-9de0-c0a2bb21c41e"}
{"level":"info","ts":"2026-07-13T15:50:03.884Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"e2415b26-5974-4863-9de0-c0a2bb21c41e","namespace":"test-ns-5xvz7","authClusters":0}
{"level":"info","ts":"2026-07-13T15:50:03.884Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"e2415b26-5974-4863-9de0-c0a2bb21c41e","namespace":"test-ns-5xvz7"}
{"level":"error","ts":"2026-07-13T15:50:03.888Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"35131f5b-90ec-4dec-9eb8-9565acf61987","issuer":"ray-selfsigned-issuer-counter-qn4mr","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-counter-qn4mr\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:03.921Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"ba5b8e00-19aa-4b29-bd84-9df077c0fdf1"}
{"level":"info","ts":"2026-07-13T15:50:03.921Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"c3cc864f-9382-4698-8d9e-94ffa383f4ed","namespacedName":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:03.921Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"c3cc864f-9382-4698-8d9e-94ffa383f4ed"}
{"level":"info","ts":"2026-07-13T15:50:03.921Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"c3cc864f-9382-4698-8d9e-94ffa383f4ed","namespace":"test-ns-5xvz7","authClusters":0}
{"level":"info","ts":"2026-07-13T15:50:03.921Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"c3cc864f-9382-4698-8d9e-94ffa383f4ed","namespace":"test-ns-5xvz7"}
{"level":"info","ts":"2026-07-13T15:50:03.922Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"5751e7ba-8f95-458e-ad7b-79c3b57a954a","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:03.922Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"5751e7ba-8f95-458e-ad7b-79c3b57a954a","finalizer":"ray.io/rayjob-finalizer"}
{"level":"error","ts":"2026-07-13T15:50:03.923Z","msg":"Failed to delete CA certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"35131f5b-90ec-4dec-9eb8-9565acf61987","certificate":"ray-ca-certificate-counter-qn4mr","error":"certificates.cert-manager.io \"ray-ca-certificate-counter-qn4mr\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:275\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:03.955Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"5751e7ba-8f95-458e-ad7b-79c3b57a954a"}
{"level":"info","ts":"2026-07-13T15:50:03.955Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"5751e7ba-8f95-458e-ad7b-79c3b57a954a","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-13T15:50:03.955Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"5751e7ba-8f95-458e-ad7b-79c3b57a954a","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-kh6fm","rayClusterName":"fail-pbfxx","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:50:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:50:03.955Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"5751e7ba-8f95-458e-ad7b-79c3b57a954a","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T15:50:03.976Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"35131f5b-90ec-4dec-9eb8-9565acf61987","errorCount":2}
{"level":"info","ts":"2026-07-13T15:50:03.976Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"35131f5b-90ec-4dec-9eb8-9565acf61987"}
{"level":"error","ts":"2026-07-13T15:50:03.977Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"35131f5b-90ec-4dec-9eb8-9565acf61987","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-counter-qn4mr\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:03.982Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"3f85ba35-53e4-4006-8e07-c45b32fb718c","namespace":"test-ns-5xvz7","clusterName":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:50:03.982Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"counter-qn4mr","reconcileID":"3f85ba35-53e4-4006-8e07-c45b32fb718c","namespace":"test-ns-5xvz7","clusterName":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:50:03.989Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"0f2ea1cf-41b8-40d9-8463-1e308c0a867b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:03.989Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"0f2ea1cf-41b8-40d9-8463-1e308c0a867b","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:03.989Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"0f2ea1cf-41b8-40d9-8463-1e308c0a867b","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:03.989Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"0f2ea1cf-41b8-40d9-8463-1e308c0a867b","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:03.996Z","logger":"raycluster-resource","msg":"default","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:03.996Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-pbfxx","namespace":"test-ns-5xvz7"}
{"level":"info","ts":"2026-07-13T15:50:04.010Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"0f2ea1cf-41b8-40d9-8463-1e308c0a867b","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:04.010Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"0f2ea1cf-41b8-40d9-8463-1e308c0a867b"}
{"level":"info","ts":"2026-07-13T15:50:04.010Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"0f2ea1cf-41b8-40d9-8463-1e308c0a867b","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:04.011Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"71392d8e-d82f-4bb0-9260-04446c86b9ed","namespacedName":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:04.021Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"71392d8e-d82f-4bb0-9260-04446c86b9ed","mode":"IntegratedOAuth","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.021Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"71392d8e-d82f-4bb0-9260-04446c86b9ed","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.021Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"71392d8e-d82f-4bb0-9260-04446c86b9ed","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.011Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"830fb498-e236-428b-b957-256e6afa4dc3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:04.011Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4934cc69-6677-44a3-b7dd-78b7c8c295bc"}
{"level":"info","ts":"2026-07-13T15:50:04.021Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4934cc69-6677-44a3-b7dd-78b7c8c295bc"}
{"level":"info","ts":"2026-07-13T15:50:04.021Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"830fb498-e236-428b-b957-256e6afa4dc3","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.021Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"830fb498-e236-428b-b957-256e6afa4dc3","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:04.011Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"8205cc20-9447-488c-8ec8-a44d7ce5ed1c","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.011Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"ab9ed43d-e6b4-45c4-abac-e5e8f73e0252","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.021Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"ab9ed43d-e6b4-45c4-abac-e5e8f73e0252","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.021Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"830fb498-e236-428b-b957-256e6afa4dc3"}
{"level":"info","ts":"2026-07-13T15:50:04.021Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"830fb498-e236-428b-b957-256e6afa4dc3","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:04.027Z","logger":"raycluster-resource","msg":"default","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.027Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-pbfxx","namespace":"test-ns-5xvz7"}
{"level":"info","ts":"2026-07-13T15:50:04.028Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4934cc69-6677-44a3-b7dd-78b7c8c295bc","name":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:04.028Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4934cc69-6677-44a3-b7dd-78b7c8c295bc","error":"head certificate secret ray-head-secret-fail-pbfxx not found"}
{"level":"error","ts":"2026-07-13T15:50:04.028Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4934cc69-6677-44a3-b7dd-78b7c8c295bc","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:04.028Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4934cc69-6677-44a3-b7dd-78b7c8c295bc","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found"}
{"level":"info","ts":"2026-07-13T15:50:04.028Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4934cc69-6677-44a3-b7dd-78b7c8c295bc","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.029Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4934cc69-6677-44a3-b7dd-78b7c8c295bc","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.029Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4934cc69-6677-44a3-b7dd-78b7c8c295bc","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.029Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4934cc69-6677-44a3-b7dd-78b7c8c295bc","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:50:04.029Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4934cc69-6677-44a3-b7dd-78b7c8c295bc","name":"fail-pbfxx","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:50:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:50:04.030Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"ab9ed43d-e6b4-45c4-abac-e5e8f73e0252","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.030Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"ab9ed43d-e6b4-45c4-abac-e5e8f73e0252","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.037Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"b1b3b70c-6ac9-4e8c-86c2-c967f5f28420","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:04.038Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"b1b3b70c-6ac9-4e8c-86c2-c967f5f28420","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.038Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"b1b3b70c-6ac9-4e8c-86c2-c967f5f28420","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:04.038Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"b1b3b70c-6ac9-4e8c-86c2-c967f5f28420"}
{"level":"info","ts":"2026-07-13T15:50:04.038Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"b1b3b70c-6ac9-4e8c-86c2-c967f5f28420","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:04.039Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"8205cc20-9447-488c-8ec8-a44d7ce5ed1c","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.039Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"0d9d8481-ced2-4441-be2b-7797aa9372ce","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.041Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"ab9ed43d-e6b4-45c4-abac-e5e8f73e0252","rayCluster":"fail-pbfxx","secret":"ca-secret"}
{"level":"info","ts":"2026-07-13T15:50:04.041Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"ab9ed43d-e6b4-45c4-abac-e5e8f73e0252","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.041Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4934cc69-6677-44a3-b7dd-78b7c8c295bc","name":"fail-pbfxx","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-pbfxx\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-5xvz7","name":"fail-pbfxx"}}
{"level":"info","ts":"2026-07-13T15:50:04.042Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4934cc69-6677-44a3-b7dd-78b7c8c295bc"}
{"level":"error","ts":"2026-07-13T15:50:04.042Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4934cc69-6677-44a3-b7dd-78b7c8c295bc","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:04.042Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4d29b0d3-4c6b-4d05-834e-93908e670500"}
{"level":"info","ts":"2026-07-13T15:50:04.042Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4d29b0d3-4c6b-4d05-834e-93908e670500"}
{"level":"info","ts":"2026-07-13T15:50:04.042Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4d29b0d3-4c6b-4d05-834e-93908e670500","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:04.042Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4d29b0d3-4c6b-4d05-834e-93908e670500","error":"head certificate secret ray-head-secret-fail-pbfxx not found"}
{"level":"error","ts":"2026-07-13T15:50:04.042Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4d29b0d3-4c6b-4d05-834e-93908e670500","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:04.042Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4d29b0d3-4c6b-4d05-834e-93908e670500","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found"}
{"level":"info","ts":"2026-07-13T15:50:04.042Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4d29b0d3-4c6b-4d05-834e-93908e670500","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.042Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4d29b0d3-4c6b-4d05-834e-93908e670500","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.042Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4d29b0d3-4c6b-4d05-834e-93908e670500","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.042Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4d29b0d3-4c6b-4d05-834e-93908e670500","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:50:04.042Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4d29b0d3-4c6b-4d05-834e-93908e670500","name":"fail-pbfxx","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:50:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:50:04.121Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"71392d8e-d82f-4bb0-9260-04446c86b9ed","name":"fail-pbfxx-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:50:04.122Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4d29b0d3-4c6b-4d05-834e-93908e670500"}
{"level":"error","ts":"2026-07-13T15:50:04.122Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"4d29b0d3-4c6b-4d05-834e-93908e670500","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:04.122Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"e4679d5e-7b6b-4103-825d-ff7178ab6409"}
{"level":"info","ts":"2026-07-13T15:50:04.122Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"e4679d5e-7b6b-4103-825d-ff7178ab6409"}
{"level":"info","ts":"2026-07-13T15:50:04.122Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"e4679d5e-7b6b-4103-825d-ff7178ab6409","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:04.122Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"e4679d5e-7b6b-4103-825d-ff7178ab6409","error":"head certificate secret ray-head-secret-fail-pbfxx not found"}
{"level":"error","ts":"2026-07-13T15:50:04.122Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"e4679d5e-7b6b-4103-825d-ff7178ab6409","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:04.122Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"e4679d5e-7b6b-4103-825d-ff7178ab6409","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found"}
{"level":"info","ts":"2026-07-13T15:50:04.122Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"e4679d5e-7b6b-4103-825d-ff7178ab6409","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.122Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"ab9ed43d-e6b4-45c4-abac-e5e8f73e0252","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.123Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"e4679d5e-7b6b-4103-825d-ff7178ab6409","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.123Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"e4679d5e-7b6b-4103-825d-ff7178ab6409","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.123Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"e4679d5e-7b6b-4103-825d-ff7178ab6409","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:50:04.123Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"e4679d5e-7b6b-4103-825d-ff7178ab6409","name":"fail-pbfxx","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:50:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:50:04.128Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"0d9d8481-ced2-4441-be2b-7797aa9372ce","name":"fail-pbfxx-head"}
{"level":"info","ts":"2026-07-13T15:50:04.128Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"71392d8e-d82f-4bb0-9260-04446c86b9ed","name":"kuberay-gateway-access","namespace":"test-ns-5xvz7","operation":"created"}
{"level":"info","ts":"2026-07-13T15:50:04.128Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"46bd2bc8-ecc2-4a37-985a-5fc5df9323a9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:04.128Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"46bd2bc8-ecc2-4a37-985a-5fc5df9323a9","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.128Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"46bd2bc8-ecc2-4a37-985a-5fc5df9323a9","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:04.128Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"46bd2bc8-ecc2-4a37-985a-5fc5df9323a9"}
{"level":"info","ts":"2026-07-13T15:50:04.128Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"46bd2bc8-ecc2-4a37-985a-5fc5df9323a9","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:04.129Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"e4679d5e-7b6b-4103-825d-ff7178ab6409","name":"fail-pbfxx","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-pbfxx\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-5xvz7","name":"fail-pbfxx"}}
{"level":"info","ts":"2026-07-13T15:50:04.129Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"e4679d5e-7b6b-4103-825d-ff7178ab6409"}
{"level":"error","ts":"2026-07-13T15:50:04.129Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"e4679d5e-7b6b-4103-825d-ff7178ab6409","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:04.131Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"ab9ed43d-e6b4-45c4-abac-e5e8f73e0252","rayCluster":"fail-pbfxx","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:50:04.133Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"d687a2d6-89f9-420b-88fc-43ce7ef42d83"}
{"level":"info","ts":"2026-07-13T15:50:04.133Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"d687a2d6-89f9-420b-88fc-43ce7ef42d83"}
{"level":"info","ts":"2026-07-13T15:50:04.133Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"d687a2d6-89f9-420b-88fc-43ce7ef42d83","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:04.133Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"d687a2d6-89f9-420b-88fc-43ce7ef42d83","error":"head certificate secret ray-head-secret-fail-pbfxx not found"}
{"level":"error","ts":"2026-07-13T15:50:04.133Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"d687a2d6-89f9-420b-88fc-43ce7ef42d83","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:04.133Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"d687a2d6-89f9-420b-88fc-43ce7ef42d83","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found"}
{"level":"info","ts":"2026-07-13T15:50:04.133Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"d687a2d6-89f9-420b-88fc-43ce7ef42d83","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.133Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"d687a2d6-89f9-420b-88fc-43ce7ef42d83","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.133Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"d687a2d6-89f9-420b-88fc-43ce7ef42d83","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.133Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"d687a2d6-89f9-420b-88fc-43ce7ef42d83"}
{"level":"error","ts":"2026-07-13T15:50:04.133Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"d687a2d6-89f9-420b-88fc-43ce7ef42d83","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:04.137Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"71392d8e-d82f-4bb0-9260-04446c86b9ed","name":"test-ns-5xvz7-fail-pbfxx","operation":"created"}
{"level":"info","ts":"2026-07-13T15:50:04.142Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"0d9d8481-ced2-4441-be2b-7797aa9372ce","name":"fail-pbfxx-workers"}
{"level":"info","ts":"2026-07-13T15:50:04.142Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"0d9d8481-ced2-4441-be2b-7797aa9372ce","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.142Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"10a94289-aae8-48fa-bb3a-7b6894aefc78","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.142Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"ab9ed43d-e6b4-45c4-abac-e5e8f73e0252","rayCluster":"fail-pbfxx","podIPs":[]}
{"level":"error","ts":"2026-07-13T15:50:04.142Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"ab9ed43d-e6b4-45c4-abac-e5e8f73e0252","error":"Certificate.cert-manager.io \"ray-worker-cert-fail-pbfxx\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:04.142Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"ab9ed43d-e6b4-45c4-abac-e5e8f73e0252"}
{"level":"error","ts":"2026-07-13T15:50:04.142Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"ab9ed43d-e6b4-45c4-abac-e5e8f73e0252","error":"Certificate.cert-manager.io \"ray-worker-cert-fail-pbfxx\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:04.142Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"cdbf6380-e492-4b23-80d8-502fb0d7f124","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.142Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"cdbf6380-e492-4b23-80d8-502fb0d7f124"}
{"level":"info","ts":"2026-07-13T15:50:04.146Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"71392d8e-d82f-4bb0-9260-04446c86b9ed","name":"kube-rbac-proxy-config-fail-pbfxx","operation":"created"}
{"level":"info","ts":"2026-07-13T15:50:04.221Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"778518cf-665a-41c0-b90f-4af41a982279","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.221Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"778518cf-665a-41c0-b90f-4af41a982279"}
{"level":"info","ts":"2026-07-13T15:50:04.221Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"a490d295-5072-4d70-8bf9-39da47a78f66"}
{"level":"info","ts":"2026-07-13T15:50:04.221Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"a490d295-5072-4d70-8bf9-39da47a78f66"}
{"level":"info","ts":"2026-07-13T15:50:04.221Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"a490d295-5072-4d70-8bf9-39da47a78f66","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:04.221Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"a490d295-5072-4d70-8bf9-39da47a78f66","error":"head certificate secret ray-head-secret-fail-pbfxx not found"}
{"level":"error","ts":"2026-07-13T15:50:04.221Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"71392d8e-d82f-4bb0-9260-04446c86b9ed","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-pbfxx\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:50:04.221Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"a490d295-5072-4d70-8bf9-39da47a78f66","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:04.221Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"a490d295-5072-4d70-8bf9-39da47a78f66","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found"}
{"level":"info","ts":"2026-07-13T15:50:04.221Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"71392d8e-d82f-4bb0-9260-04446c86b9ed"}
{"level":"error","ts":"2026-07-13T15:50:04.221Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"71392d8e-d82f-4bb0-9260-04446c86b9ed","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-pbfxx\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:04.221Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"a490d295-5072-4d70-8bf9-39da47a78f66","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.221Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"a490d295-5072-4d70-8bf9-39da47a78f66","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.221Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"8d401ddc-f62e-4774-aecc-7ebafa9abef1","namespacedName":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:04.222Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"a490d295-5072-4d70-8bf9-39da47a78f66","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.222Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"a490d295-5072-4d70-8bf9-39da47a78f66"}
{"level":"info","ts":"2026-07-13T15:50:04.222Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"8d401ddc-f62e-4774-aecc-7ebafa9abef1","mode":"IntegratedOAuth","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.222Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"8d401ddc-f62e-4774-aecc-7ebafa9abef1","cluster":"fail-pbfxx"}
{"level":"error","ts":"2026-07-13T15:50:04.222Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"a490d295-5072-4d70-8bf9-39da47a78f66","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:04.226Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"10a94289-aae8-48fa-bb3a-7b6894aefc78","name":"fail-pbfxx-head"}
{"level":"info","ts":"2026-07-13T15:50:04.234Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"8d401ddc-f62e-4774-aecc-7ebafa9abef1","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.234Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"125a9d9a-24a4-4819-af33-b2e5be1f31c8","namespacedName":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:04.235Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"125a9d9a-24a4-4819-af33-b2e5be1f31c8","mode":"IntegratedOAuth","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.235Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"125a9d9a-24a4-4819-af33-b2e5be1f31c8","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.235Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"389f30fa-1627-4b37-bd2c-d747f6905425","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.321Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"389f30fa-1627-4b37-bd2c-d747f6905425"}
{"level":"info","ts":"2026-07-13T15:50:04.235Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"b41ff9a0-66c9-492b-b740-8e3dbab00330","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:04.321Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"b41ff9a0-66c9-492b-b740-8e3dbab00330","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.321Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"b41ff9a0-66c9-492b-b740-8e3dbab00330","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:04.321Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"b41ff9a0-66c9-492b-b740-8e3dbab00330"}
{"level":"info","ts":"2026-07-13T15:50:04.321Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"b41ff9a0-66c9-492b-b740-8e3dbab00330","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:04.242Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"10a94289-aae8-48fa-bb3a-7b6894aefc78","name":"fail-pbfxx-workers"}
{"level":"info","ts":"2026-07-13T15:50:04.302Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc706c5-d5d5-4305-be23-5637f3b42f4a"}
{"level":"info","ts":"2026-07-13T15:50:04.322Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc706c5-d5d5-4305-be23-5637f3b42f4a"}
{"level":"info","ts":"2026-07-13T15:50:04.322Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"10a94289-aae8-48fa-bb3a-7b6894aefc78","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.322Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc706c5-d5d5-4305-be23-5637f3b42f4a","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:04.322Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc706c5-d5d5-4305-be23-5637f3b42f4a","error":"head certificate secret ray-head-secret-fail-pbfxx not found"}
{"level":"error","ts":"2026-07-13T15:50:04.322Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc706c5-d5d5-4305-be23-5637f3b42f4a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:04.322Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc706c5-d5d5-4305-be23-5637f3b42f4a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found"}
{"level":"info","ts":"2026-07-13T15:50:04.322Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"482148ae-6eef-4797-b025-d255147549da","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.322Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc706c5-d5d5-4305-be23-5637f3b42f4a","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.322Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc706c5-d5d5-4305-be23-5637f3b42f4a","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.322Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc706c5-d5d5-4305-be23-5637f3b42f4a","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.322Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc706c5-d5d5-4305-be23-5637f3b42f4a"}
{"level":"error","ts":"2026-07-13T15:50:04.322Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc706c5-d5d5-4305-be23-5637f3b42f4a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:50:04.339Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"125a9d9a-24a4-4819-af33-b2e5be1f31c8","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-pbfxx\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:04.339Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"125a9d9a-24a4-4819-af33-b2e5be1f31c8"}
{"level":"error","ts":"2026-07-13T15:50:04.339Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"125a9d9a-24a4-4819-af33-b2e5be1f31c8","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-pbfxx\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:04.345Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"30997dfb-428e-448f-9784-ed9312df27d4","namespacedName":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:04.345Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"30997dfb-428e-448f-9784-ed9312df27d4","mode":"IntegratedOAuth","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.345Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"30997dfb-428e-448f-9784-ed9312df27d4","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.350Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"482148ae-6eef-4797-b025-d255147549da","name":"fail-pbfxx-head"}
{"level":"info","ts":"2026-07-13T15:50:04.354Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"30997dfb-428e-448f-9784-ed9312df27d4","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.363Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"482148ae-6eef-4797-b025-d255147549da","name":"fail-pbfxx-workers"}
{"level":"info","ts":"2026-07-13T15:50:04.363Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"482148ae-6eef-4797-b025-d255147549da","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:04.483Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"977b9252-b261-4195-8f86-98da93671e8f"}
{"level":"info","ts":"2026-07-13T15:50:04.483Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"977b9252-b261-4195-8f86-98da93671e8f"}
{"level":"info","ts":"2026-07-13T15:50:04.483Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"977b9252-b261-4195-8f86-98da93671e8f","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:04.483Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"977b9252-b261-4195-8f86-98da93671e8f","error":"head certificate secret ray-head-secret-fail-pbfxx not found"}
{"level":"error","ts":"2026-07-13T15:50:04.483Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"977b9252-b261-4195-8f86-98da93671e8f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:04.483Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"977b9252-b261-4195-8f86-98da93671e8f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found"}
{"level":"info","ts":"2026-07-13T15:50:04.483Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"977b9252-b261-4195-8f86-98da93671e8f","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.483Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"977b9252-b261-4195-8f86-98da93671e8f","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.483Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"977b9252-b261-4195-8f86-98da93671e8f","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.483Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"977b9252-b261-4195-8f86-98da93671e8f"}
{"level":"error","ts":"2026-07-13T15:50:04.483Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"977b9252-b261-4195-8f86-98da93671e8f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:04.803Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"df27d56e-8661-4e22-a7f4-6e6dfacd3627"}
{"level":"info","ts":"2026-07-13T15:50:04.803Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"df27d56e-8661-4e22-a7f4-6e6dfacd3627"}
{"level":"info","ts":"2026-07-13T15:50:04.803Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"df27d56e-8661-4e22-a7f4-6e6dfacd3627","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:04.803Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"df27d56e-8661-4e22-a7f4-6e6dfacd3627","error":"head certificate secret ray-head-secret-fail-pbfxx not found"}
{"level":"error","ts":"2026-07-13T15:50:04.803Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"df27d56e-8661-4e22-a7f4-6e6dfacd3627","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:04.803Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"df27d56e-8661-4e22-a7f4-6e6dfacd3627","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found"}
{"level":"info","ts":"2026-07-13T15:50:04.803Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"df27d56e-8661-4e22-a7f4-6e6dfacd3627","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.804Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"df27d56e-8661-4e22-a7f4-6e6dfacd3627","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.804Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"df27d56e-8661-4e22-a7f4-6e6dfacd3627","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:04.804Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"df27d56e-8661-4e22-a7f4-6e6dfacd3627"}
{"level":"error","ts":"2026-07-13T15:50:04.804Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"df27d56e-8661-4e22-a7f4-6e6dfacd3627","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:05.444Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"497c634e-8363-42cb-accf-4f38dfe14ee0"}
{"level":"info","ts":"2026-07-13T15:50:05.444Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"497c634e-8363-42cb-accf-4f38dfe14ee0"}
{"level":"info","ts":"2026-07-13T15:50:05.444Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"497c634e-8363-42cb-accf-4f38dfe14ee0","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:05.444Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"497c634e-8363-42cb-accf-4f38dfe14ee0","error":"head certificate secret ray-head-secret-fail-pbfxx not found"}
{"level":"error","ts":"2026-07-13T15:50:05.444Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"497c634e-8363-42cb-accf-4f38dfe14ee0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:05.444Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"497c634e-8363-42cb-accf-4f38dfe14ee0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found"}
{"level":"info","ts":"2026-07-13T15:50:05.444Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"497c634e-8363-42cb-accf-4f38dfe14ee0","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:05.444Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"497c634e-8363-42cb-accf-4f38dfe14ee0","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:05.444Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"497c634e-8363-42cb-accf-4f38dfe14ee0","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:50:05.444Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"497c634e-8363-42cb-accf-4f38dfe14ee0"}
{"level":"error","ts":"2026-07-13T15:50:05.444Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"497c634e-8363-42cb-accf-4f38dfe14ee0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-pbfxx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:06.725Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf"}
{"level":"info","ts":"2026-07-13T15:50:06.725Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf"}
{"level":"info","ts":"2026-07-13T15:50:06.725Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:06.725Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf"}
{"level":"info","ts":"2026-07-13T15:50:06.725Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:50:06.725Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf"}
{"level":"info","ts":"2026-07-13T15:50:06.725Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","mode":"IntegratedOAuth","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:06.725Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","cluster":"fail-pbfxx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:50:06.725Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","cluster":"fail-pbfxx","authType":"IntegratedOAuth","serviceAccount":"fail-pbfxx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T15:50:06.725Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-pbfxx","ray.io/group":"headgroup","ray.io/identifier":"fail-pbfxx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T15:50:06.725Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T15:50:06.725Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:50:06.725Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:50:06.725Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","enabled":true}
{"level":"info","ts":"2026-07-13T15:50:06.740Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","name":"fail-pbfxx-head-27cb9"}
{"level":"info","ts":"2026-07-13T15:50:06.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:50:06.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:06.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T15:50:06.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:06.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T15:50:06.740Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf"}
{"level":"info","ts":"2026-07-13T15:50:06.740Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","nodeType":"worker","rayStartParams":{"address":"fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T15:50:06.740Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","rayStartCmd":"ray start  --address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:50:06.740Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:50:06.740Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","enabled":true}
{"level":"info","ts":"2026-07-13T15:50:06.754Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","name":"fail-pbfxx-small-group-worker-bhr2g"}
{"level":"info","ts":"2026-07-13T15:50:06.755Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","podName":"fail-pbfxx-head-27cb9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:06.755Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-pbfxx-head-svc"},"newHead":{"podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"}}
{"level":"info","ts":"2026-07-13T15:50:06.755Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"53df1ad8-8755-4345-9a52-fa592d1d8edf","name":"fail-pbfxx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:50:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:50:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:50:06.823Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"42a636f2-c44d-4b5f-9bf1-905f69ce9976","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:06.823Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"42a636f2-c44d-4b5f-9bf1-905f69ce9976","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:06.823Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"42a636f2-c44d-4b5f-9bf1-905f69ce9976","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:06.823Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"42a636f2-c44d-4b5f-9bf1-905f69ce9976"}
{"level":"info","ts":"2026-07-13T15:50:06.823Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"42a636f2-c44d-4b5f-9bf1-905f69ce9976","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:06.823Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"13898f06-5e33-48c5-abdb-ff5e69b249d6","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:06.823Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"13898f06-5e33-48c5-abdb-ff5e69b249d6","rayCluster":"fail-pbfxx","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:50:06.823Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"2579d90e-7467-48e1-ba50-8d2830dd8c77","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:06.830Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ac589d03-434a-4196-9743-4db054396567","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:06.830Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ac589d03-434a-4196-9743-4db054396567","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 15:50:03 +0000 UTC","Now":"2026-07-13T15:50:06.830Z","ShutdownTime":"2026-07-13T15:50:03.000Z"}
{"level":"info","ts":"2026-07-13T15:50:06.830Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ac589d03-434a-4196-9743-4db054396567","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T15:50:06.830Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ac589d03-434a-4196-9743-4db054396567","RayCluster":{"name":"counter-qn4mr","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:06.830Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ac589d03-434a-4196-9743-4db054396567","isClusterDeleted":true}
{"level":"info","ts":"2026-07-13T15:50:06.830Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"ac589d03-434a-4196-9743-4db054396567","RayCluster":"counter-qn4mr"}
{"level":"info","ts":"2026-07-13T15:50:06.921Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cf779800-184d-448e-b526-3a7629a2a32d"}
{"level":"info","ts":"2026-07-13T15:50:06.921Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cf779800-184d-448e-b526-3a7629a2a32d"}
{"level":"info","ts":"2026-07-13T15:50:06.921Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cf779800-184d-448e-b526-3a7629a2a32d","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:06.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cf779800-184d-448e-b526-3a7629a2a32d","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:50:06.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cf779800-184d-448e-b526-3a7629a2a32d","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:06.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cf779800-184d-448e-b526-3a7629a2a32d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:50:06.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cf779800-184d-448e-b526-3a7629a2a32d","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:06.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cf779800-184d-448e-b526-3a7629a2a32d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:06.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cf779800-184d-448e-b526-3a7629a2a32d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:50:06.922Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cf779800-184d-448e-b526-3a7629a2a32d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:06.922Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cf779800-184d-448e-b526-3a7629a2a32d","podName":"fail-pbfxx-head-27cb9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:06.922Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cf779800-184d-448e-b526-3a7629a2a32d","podName":"fail-pbfxx-head-27cb9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:06.922Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cf779800-184d-448e-b526-3a7629a2a32d","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:50:06.922Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cf779800-184d-448e-b526-3a7629a2a32d","name":"fail-pbfxx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:50:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:50:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:50:06.925Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"2579d90e-7467-48e1-ba50-8d2830dd8c77","name":"fail-pbfxx-head"}
{"level":"info","ts":"2026-07-13T15:50:06.935Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8591f3bb-c962-4980-bdac-1c85cca12f1a"}
{"level":"info","ts":"2026-07-13T15:50:06.935Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8591f3bb-c962-4980-bdac-1c85cca12f1a"}
{"level":"info","ts":"2026-07-13T15:50:06.935Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8591f3bb-c962-4980-bdac-1c85cca12f1a","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:07.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8591f3bb-c962-4980-bdac-1c85cca12f1a","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:50:07.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8591f3bb-c962-4980-bdac-1c85cca12f1a","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:07.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8591f3bb-c962-4980-bdac-1c85cca12f1a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:50:07.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8591f3bb-c962-4980-bdac-1c85cca12f1a","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:07.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8591f3bb-c962-4980-bdac-1c85cca12f1a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:07.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8591f3bb-c962-4980-bdac-1c85cca12f1a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:50:07.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8591f3bb-c962-4980-bdac-1c85cca12f1a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:07.021Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"2579d90e-7467-48e1-ba50-8d2830dd8c77","name":"fail-pbfxx-workers"}
{"level":"info","ts":"2026-07-13T15:50:07.021Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"2579d90e-7467-48e1-ba50-8d2830dd8c77","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:07.021Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8591f3bb-c962-4980-bdac-1c85cca12f1a","podName":"fail-pbfxx-head-27cb9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:07.021Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8591f3bb-c962-4980-bdac-1c85cca12f1a","podName":"fail-pbfxx-head-27cb9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:07.021Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"6b56d785-ff28-4c79-80cc-f7f560ba4146","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:07.021Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"6b56d785-ff28-4c79-80cc-f7f560ba4146","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:07.021Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"6b56d785-ff28-4c79-80cc-f7f560ba4146","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:07.021Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8591f3bb-c962-4980-bdac-1c85cca12f1a","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:50:07.021Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8591f3bb-c962-4980-bdac-1c85cca12f1a","name":"fail-pbfxx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:50:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:50:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:50:07.021Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"6b56d785-ff28-4c79-80cc-f7f560ba4146"}
{"level":"info","ts":"2026-07-13T15:50:07.021Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"6b56d785-ff28-4c79-80cc-f7f560ba4146","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:07.022Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"ea553034-cda3-4d24-bc8f-61a99bae7441","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:07.022Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"f2a792d1-2c0d-4625-a934-afe996c9cd74","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:07.022Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"2defb8b3-6af7-4844-bde9-beb02e3f1b6b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:07.022Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"2defb8b3-6af7-4844-bde9-beb02e3f1b6b","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:07.022Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"ea553034-cda3-4d24-bc8f-61a99bae7441","rayCluster":"fail-pbfxx","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:50:07.022Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"2defb8b3-6af7-4844-bde9-beb02e3f1b6b","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:07.022Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"2defb8b3-6af7-4844-bde9-beb02e3f1b6b"}
{"level":"info","ts":"2026-07-13T15:50:07.022Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"2defb8b3-6af7-4844-bde9-beb02e3f1b6b","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:07.028Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8591f3bb-c962-4980-bdac-1c85cca12f1a","name":"fail-pbfxx","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-pbfxx\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-5xvz7","name":"fail-pbfxx"}}
{"level":"info","ts":"2026-07-13T15:50:07.028Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8591f3bb-c962-4980-bdac-1c85cca12f1a"}
{"level":"error","ts":"2026-07-13T15:50:07.028Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8591f3bb-c962-4980-bdac-1c85cca12f1a","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-pbfxx\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:50:07.031Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"f2a792d1-2c0d-4625-a934-afe996c9cd74","name":"fail-pbfxx-head"}
{"level":"info","ts":"2026-07-13T15:50:07.034Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cfd55781-9ac5-4f19-b66c-e5376734716b"}
{"level":"info","ts":"2026-07-13T15:50:07.034Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cfd55781-9ac5-4f19-b66c-e5376734716b"}
{"level":"info","ts":"2026-07-13T15:50:07.034Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cfd55781-9ac5-4f19-b66c-e5376734716b","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:07.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cfd55781-9ac5-4f19-b66c-e5376734716b","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:50:07.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cfd55781-9ac5-4f19-b66c-e5376734716b","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:07.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cfd55781-9ac5-4f19-b66c-e5376734716b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:50:07.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cfd55781-9ac5-4f19-b66c-e5376734716b","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:07.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cfd55781-9ac5-4f19-b66c-e5376734716b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:07.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cfd55781-9ac5-4f19-b66c-e5376734716b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:50:07.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cfd55781-9ac5-4f19-b66c-e5376734716b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:07.034Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cfd55781-9ac5-4f19-b66c-e5376734716b","podName":"fail-pbfxx-head-27cb9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:07.034Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cfd55781-9ac5-4f19-b66c-e5376734716b","podName":"fail-pbfxx-head-27cb9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:07.034Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cfd55781-9ac5-4f19-b66c-e5376734716b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:50:07.034Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cfd55781-9ac5-4f19-b66c-e5376734716b","seconds":300}
{"level":"info","ts":"2026-07-13T15:50:07.042Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"f2a792d1-2c0d-4625-a934-afe996c9cd74","name":"fail-pbfxx-workers"}
{"level":"info","ts":"2026-07-13T15:50:07.042Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"f2a792d1-2c0d-4625-a934-afe996c9cd74","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:07.208Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"017e97b1-edc6-4d68-b349-b01d727f1b59"}
{"level":"info","ts":"2026-07-13T15:50:07.208Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"017e97b1-edc6-4d68-b349-b01d727f1b59"}
{"level":"info","ts":"2026-07-13T15:50:07.208Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"017e97b1-edc6-4d68-b349-b01d727f1b59","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:07.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"017e97b1-edc6-4d68-b349-b01d727f1b59","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:50:07.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"017e97b1-edc6-4d68-b349-b01d727f1b59","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:07.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"017e97b1-edc6-4d68-b349-b01d727f1b59","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:50:07.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"017e97b1-edc6-4d68-b349-b01d727f1b59","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:07.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"017e97b1-edc6-4d68-b349-b01d727f1b59","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:07.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"017e97b1-edc6-4d68-b349-b01d727f1b59","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:50:07.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"017e97b1-edc6-4d68-b349-b01d727f1b59","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:07.209Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"017e97b1-edc6-4d68-b349-b01d727f1b59","podName":"fail-pbfxx-head-27cb9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:07.209Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"017e97b1-edc6-4d68-b349-b01d727f1b59","podName":"fail-pbfxx-head-27cb9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:07.209Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"017e97b1-edc6-4d68-b349-b01d727f1b59","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:50:07.209Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"017e97b1-edc6-4d68-b349-b01d727f1b59","seconds":300}
{"level":"info","ts":"2026-07-13T15:50:07.211Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5a69cbbb-758f-41c5-b594-6e072b9a4a39"}
{"level":"info","ts":"2026-07-13T15:50:07.211Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5a69cbbb-758f-41c5-b594-6e072b9a4a39"}
{"level":"info","ts":"2026-07-13T15:50:07.211Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5a69cbbb-758f-41c5-b594-6e072b9a4a39","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:07.211Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5a69cbbb-758f-41c5-b594-6e072b9a4a39","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:50:07.211Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5a69cbbb-758f-41c5-b594-6e072b9a4a39","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:07.211Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5a69cbbb-758f-41c5-b594-6e072b9a4a39","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:50:07.211Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5a69cbbb-758f-41c5-b594-6e072b9a4a39","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:07.211Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5a69cbbb-758f-41c5-b594-6e072b9a4a39","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:07.211Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5a69cbbb-758f-41c5-b594-6e072b9a4a39","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:50:07.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5a69cbbb-758f-41c5-b594-6e072b9a4a39","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:07.212Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5a69cbbb-758f-41c5-b594-6e072b9a4a39","podName":"fail-pbfxx-head-27cb9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:07.212Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5a69cbbb-758f-41c5-b594-6e072b9a4a39","podName":"fail-pbfxx-head-27cb9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:07.212Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5a69cbbb-758f-41c5-b594-6e072b9a4a39","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:50:07.212Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5a69cbbb-758f-41c5-b594-6e072b9a4a39","seconds":300}
{"level":"info","ts":"2026-07-13T15:50:07.935Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7e2b92ce-6547-4c01-9903-6123e0a962c8"}
{"level":"info","ts":"2026-07-13T15:50:07.935Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7e2b92ce-6547-4c01-9903-6123e0a962c8"}
{"level":"info","ts":"2026-07-13T15:50:07.935Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7e2b92ce-6547-4c01-9903-6123e0a962c8","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:07.935Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7e2b92ce-6547-4c01-9903-6123e0a962c8","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:50:07.935Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7e2b92ce-6547-4c01-9903-6123e0a962c8","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:07.935Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7e2b92ce-6547-4c01-9903-6123e0a962c8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:50:07.936Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7e2b92ce-6547-4c01-9903-6123e0a962c8","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:07.936Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7e2b92ce-6547-4c01-9903-6123e0a962c8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:07.936Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7e2b92ce-6547-4c01-9903-6123e0a962c8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:50:07.936Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7e2b92ce-6547-4c01-9903-6123e0a962c8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:07.936Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7e2b92ce-6547-4c01-9903-6123e0a962c8","podName":"fail-pbfxx-head-27cb9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:07.936Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7e2b92ce-6547-4c01-9903-6123e0a962c8","podName":"fail-pbfxx-head-27cb9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:07.936Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7e2b92ce-6547-4c01-9903-6123e0a962c8","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"newHead":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"}}
{"level":"info","ts":"2026-07-13T15:50:07.936Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7e2b92ce-6547-4c01-9903-6123e0a962c8","name":"fail-pbfxx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:50:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:50:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:50:07.947Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"464cabf5-6383-43a7-b9ce-1247279c4987","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:07.948Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"8fdf83d9-8a91-467f-a5a4-d266cb9ea1db","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:07.948Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"464cabf5-6383-43a7-b9ce-1247279c4987","pod":"fail-pbfxx-head-27cb9","ip":"10.133.0.20"}
{"level":"info","ts":"2026-07-13T15:50:07.948Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"2e321f1c-7a16-4713-b79f-cb23e921bf65","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:07.948Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"2e321f1c-7a16-4713-b79f-cb23e921bf65","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:07.948Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"2e321f1c-7a16-4713-b79f-cb23e921bf65","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:07.948Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"2e321f1c-7a16-4713-b79f-cb23e921bf65"}
{"level":"info","ts":"2026-07-13T15:50:07.948Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"2e321f1c-7a16-4713-b79f-cb23e921bf65","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:07.960Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"464cabf5-6383-43a7-b9ce-1247279c4987","certificate":"ray-head-cert-fail-pbfxx","dnsNames":["fail-pbfxx-head-svc","localhost","fail-pbfxx-head-svc.test-ns-5xvz7.svc","fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local"],"ipAddresses":["10.133.0.20","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:50:07.965Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"8fdf83d9-8a91-467f-a5a4-d266cb9ea1db","name":"fail-pbfxx-head"}
{"level":"info","ts":"2026-07-13T15:50:07.972Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"464cabf5-6383-43a7-b9ce-1247279c4987","certificate":"ray-worker-cert-fail-pbfxx","dnsNames":["fail-pbfxx-worker-svc","localhost","fail-pbfxx-small-group","fail-pbfxx-worker-svc.test-ns-5xvz7.svc","fail-pbfxx-worker-svc.test-ns-5xvz7.svc.cluster.local","*.fail-pbfxx-worker-svc.test-ns-5xvz7.svc","*.fail-pbfxx-worker-svc.test-ns-5xvz7.svc.cluster.local","*-worker-*.test-ns-5xvz7.svc","*-worker-*.test-ns-5xvz7.svc.cluster.local"],"ipAddresses":["10.133.0.20","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:50:07.972Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"464cabf5-6383-43a7-b9ce-1247279c4987","rayCluster":"fail-pbfxx","podIPs":["10.133.0.20"]}
{"level":"info","ts":"2026-07-13T15:50:07.979Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"8fdf83d9-8a91-467f-a5a4-d266cb9ea1db","name":"fail-pbfxx-workers"}
{"level":"info","ts":"2026-07-13T15:50:07.979Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"8fdf83d9-8a91-467f-a5a4-d266cb9ea1db","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:08.016Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"459cdb8a-6818-4ff7-8469-c3da8502da1c"}
{"level":"info","ts":"2026-07-13T15:50:08.016Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"459cdb8a-6818-4ff7-8469-c3da8502da1c"}
{"level":"info","ts":"2026-07-13T15:50:08.016Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"459cdb8a-6818-4ff7-8469-c3da8502da1c","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:08.016Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"459cdb8a-6818-4ff7-8469-c3da8502da1c","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:50:08.016Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"459cdb8a-6818-4ff7-8469-c3da8502da1c","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:08.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"459cdb8a-6818-4ff7-8469-c3da8502da1c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:50:08.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"459cdb8a-6818-4ff7-8469-c3da8502da1c","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:08.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"459cdb8a-6818-4ff7-8469-c3da8502da1c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:08.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"459cdb8a-6818-4ff7-8469-c3da8502da1c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:50:08.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"459cdb8a-6818-4ff7-8469-c3da8502da1c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:08.017Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"459cdb8a-6818-4ff7-8469-c3da8502da1c","podName":"fail-pbfxx-head-27cb9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:08.017Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"459cdb8a-6818-4ff7-8469-c3da8502da1c","podName":"fail-pbfxx-head-27cb9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:08.017Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"459cdb8a-6818-4ff7-8469-c3da8502da1c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:50:08.017Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"459cdb8a-6818-4ff7-8469-c3da8502da1c","seconds":300}
{"level":"info","ts":"2026-07-13T15:50:09.947Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2b3092d4-8e23-4a5e-ab86-01f808e84512"}
{"level":"info","ts":"2026-07-13T15:50:09.947Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2b3092d4-8e23-4a5e-ab86-01f808e84512"}
{"level":"info","ts":"2026-07-13T15:50:09.947Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2b3092d4-8e23-4a5e-ab86-01f808e84512","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:09.947Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2b3092d4-8e23-4a5e-ab86-01f808e84512","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:50:09.947Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2b3092d4-8e23-4a5e-ab86-01f808e84512","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:09.947Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2b3092d4-8e23-4a5e-ab86-01f808e84512","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:50:09.947Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2b3092d4-8e23-4a5e-ab86-01f808e84512","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:09.947Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2b3092d4-8e23-4a5e-ab86-01f808e84512","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:09.947Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2b3092d4-8e23-4a5e-ab86-01f808e84512","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:50:09.947Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2b3092d4-8e23-4a5e-ab86-01f808e84512","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:09.948Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2b3092d4-8e23-4a5e-ab86-01f808e84512","podName":"fail-pbfxx-head-27cb9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:09.948Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2b3092d4-8e23-4a5e-ab86-01f808e84512","podName":"fail-pbfxx-head-27cb9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:09.948Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2b3092d4-8e23-4a5e-ab86-01f808e84512","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:50:09.948Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2b3092d4-8e23-4a5e-ab86-01f808e84512","seconds":300}
{"level":"info","ts":"2026-07-13T15:50:10.022Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"9d41fce9-3b84-4ef5-b0da-d19df956ad76","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:10.022Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"9d41fce9-3b84-4ef5-b0da-d19df956ad76","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:10.022Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"9d41fce9-3b84-4ef5-b0da-d19df956ad76","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:10.023Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"9d41fce9-3b84-4ef5-b0da-d19df956ad76"}
{"level":"info","ts":"2026-07-13T15:50:10.023Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"9d41fce9-3b84-4ef5-b0da-d19df956ad76","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:12.953Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6e6e49c5-3369-4364-94c3-1493dcde0689"}
{"level":"info","ts":"2026-07-13T15:50:12.953Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6e6e49c5-3369-4364-94c3-1493dcde0689"}
{"level":"info","ts":"2026-07-13T15:50:12.953Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6e6e49c5-3369-4364-94c3-1493dcde0689","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:12.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6e6e49c5-3369-4364-94c3-1493dcde0689","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:50:12.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6e6e49c5-3369-4364-94c3-1493dcde0689","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:12.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6e6e49c5-3369-4364-94c3-1493dcde0689","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:50:12.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6e6e49c5-3369-4364-94c3-1493dcde0689","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:12.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6e6e49c5-3369-4364-94c3-1493dcde0689","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:12.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6e6e49c5-3369-4364-94c3-1493dcde0689","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:50:12.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6e6e49c5-3369-4364-94c3-1493dcde0689","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:12.953Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6e6e49c5-3369-4364-94c3-1493dcde0689","podName":"fail-pbfxx-head-27cb9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:12.953Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6e6e49c5-3369-4364-94c3-1493dcde0689","podName":"fail-pbfxx-head-27cb9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:12.953Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6e6e49c5-3369-4364-94c3-1493dcde0689","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:50:12.953Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6e6e49c5-3369-4364-94c3-1493dcde0689","seconds":300}
{"level":"info","ts":"2026-07-13T15:50:13.024Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"017b28e5-5bfa-4c09-af70-ea594dc70f83","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:13.024Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"017b28e5-5bfa-4c09-af70-ea594dc70f83","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:13.024Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"017b28e5-5bfa-4c09-af70-ea594dc70f83","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:13.024Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"017b28e5-5bfa-4c09-af70-ea594dc70f83"}
{"level":"info","ts":"2026-07-13T15:50:13.024Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"017b28e5-5bfa-4c09-af70-ea594dc70f83","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:13.958Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"00ee781b-4734-428b-bcfe-3db07a01c395"}
{"level":"info","ts":"2026-07-13T15:50:13.958Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"00ee781b-4734-428b-bcfe-3db07a01c395"}
{"level":"info","ts":"2026-07-13T15:50:13.958Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"00ee781b-4734-428b-bcfe-3db07a01c395","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:13.958Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"00ee781b-4734-428b-bcfe-3db07a01c395","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:50:13.958Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"00ee781b-4734-428b-bcfe-3db07a01c395","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:13.958Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"00ee781b-4734-428b-bcfe-3db07a01c395","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:50:13.958Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"00ee781b-4734-428b-bcfe-3db07a01c395","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:13.958Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"00ee781b-4734-428b-bcfe-3db07a01c395","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:13.958Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"00ee781b-4734-428b-bcfe-3db07a01c395","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:50:13.958Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"00ee781b-4734-428b-bcfe-3db07a01c395","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:13.958Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"00ee781b-4734-428b-bcfe-3db07a01c395","podName":"fail-pbfxx-head-27cb9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:50:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:50:13.959Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"00ee781b-4734-428b-bcfe-3db07a01c395","podName":"fail-pbfxx-head-27cb9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:50:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:50:13.959Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"00ee781b-4734-428b-bcfe-3db07a01c395","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:50:13.959Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"00ee781b-4734-428b-bcfe-3db07a01c395","name":"fail-pbfxx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:50:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:50:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:50:13.971Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"b84a3bbf-3936-4df3-aeff-ad3c24f27033","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:13.971Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"3d4e30ea-7876-4826-bb50-3b3e0bdca821","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:13.971Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"3d4e30ea-7876-4826-bb50-3b3e0bdca821","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:13.971Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"3d4e30ea-7876-4826-bb50-3b3e0bdca821","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:13.971Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"2f163a4c-fb91-415c-8795-432e81e2f9c6","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:13.971Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"b84a3bbf-3936-4df3-aeff-ad3c24f27033","pod":"fail-pbfxx-head-27cb9","ip":"10.133.0.20"}
{"level":"info","ts":"2026-07-13T15:50:13.971Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"b84a3bbf-3936-4df3-aeff-ad3c24f27033","pod":"fail-pbfxx-small-group-worker-bhr2g","ip":"10.134.0.26"}
{"level":"info","ts":"2026-07-13T15:50:13.971Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"3d4e30ea-7876-4826-bb50-3b3e0bdca821"}
{"level":"info","ts":"2026-07-13T15:50:13.971Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"3d4e30ea-7876-4826-bb50-3b3e0bdca821","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:13.983Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"2f163a4c-fb91-415c-8795-432e81e2f9c6","name":"fail-pbfxx-head"}
{"level":"info","ts":"2026-07-13T15:50:13.983Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"b84a3bbf-3936-4df3-aeff-ad3c24f27033","certificate":"ray-head-cert-fail-pbfxx","dnsNames":["fail-pbfxx-head-svc","localhost","fail-pbfxx-head-svc.test-ns-5xvz7.svc","fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local"],"ipAddresses":["10.133.0.20","10.134.0.26","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:50:13.995Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"2f163a4c-fb91-415c-8795-432e81e2f9c6","name":"fail-pbfxx-workers"}
{"level":"info","ts":"2026-07-13T15:50:13.995Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"2f163a4c-fb91-415c-8795-432e81e2f9c6","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:13.995Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"b84a3bbf-3936-4df3-aeff-ad3c24f27033","certificate":"ray-worker-cert-fail-pbfxx","dnsNames":["fail-pbfxx-worker-svc","localhost","fail-pbfxx-small-group","fail-pbfxx-worker-svc.test-ns-5xvz7.svc","fail-pbfxx-worker-svc.test-ns-5xvz7.svc.cluster.local","*.fail-pbfxx-worker-svc.test-ns-5xvz7.svc","*.fail-pbfxx-worker-svc.test-ns-5xvz7.svc.cluster.local","*-worker-*.test-ns-5xvz7.svc","*-worker-*.test-ns-5xvz7.svc.cluster.local"],"ipAddresses":["10.133.0.20","10.134.0.26","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:50:13.995Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"b84a3bbf-3936-4df3-aeff-ad3c24f27033","rayCluster":"fail-pbfxx","podIPs":["10.133.0.20","10.134.0.26"]}
{"level":"info","ts":"2026-07-13T15:50:15.971Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"487379a7-7010-45a7-9ed6-ddcffe8c3e49"}
{"level":"info","ts":"2026-07-13T15:50:15.971Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"487379a7-7010-45a7-9ed6-ddcffe8c3e49"}
{"level":"info","ts":"2026-07-13T15:50:15.971Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"487379a7-7010-45a7-9ed6-ddcffe8c3e49","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:15.971Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"487379a7-7010-45a7-9ed6-ddcffe8c3e49","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:50:15.971Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"487379a7-7010-45a7-9ed6-ddcffe8c3e49","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:15.971Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"487379a7-7010-45a7-9ed6-ddcffe8c3e49","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:50:15.971Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"487379a7-7010-45a7-9ed6-ddcffe8c3e49","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:15.971Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"487379a7-7010-45a7-9ed6-ddcffe8c3e49","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:15.971Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"487379a7-7010-45a7-9ed6-ddcffe8c3e49","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:50:15.971Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"487379a7-7010-45a7-9ed6-ddcffe8c3e49","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:15.971Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"487379a7-7010-45a7-9ed6-ddcffe8c3e49","podName":"fail-pbfxx-small-group-worker-bhr2g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:15.972Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"487379a7-7010-45a7-9ed6-ddcffe8c3e49","podName":"fail-pbfxx-small-group-worker-bhr2g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:15.972Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"487379a7-7010-45a7-9ed6-ddcffe8c3e49","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:50:15.972Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"487379a7-7010-45a7-9ed6-ddcffe8c3e49","seconds":300}
{"level":"info","ts":"2026-07-13T15:50:16.024Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"589a1782-dee1-484c-9277-0f15b809d7ac","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:16.025Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"589a1782-dee1-484c-9277-0f15b809d7ac","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:16.025Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"589a1782-dee1-484c-9277-0f15b809d7ac","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:16.025Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"589a1782-dee1-484c-9277-0f15b809d7ac"}
{"level":"info","ts":"2026-07-13T15:50:16.025Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"589a1782-dee1-484c-9277-0f15b809d7ac","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:19.033Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"7680848e-f95e-42c1-ae64-a7185ba844a2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:19.033Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"7680848e-f95e-42c1-ae64-a7185ba844a2","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:19.033Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"7680848e-f95e-42c1-ae64-a7185ba844a2","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:19.034Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"7680848e-f95e-42c1-ae64-a7185ba844a2"}
{"level":"info","ts":"2026-07-13T15:50:19.034Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"7680848e-f95e-42c1-ae64-a7185ba844a2","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:22.034Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"c3846c83-886f-4bf1-b016-f0e996cdd5b2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:22.034Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"c3846c83-886f-4bf1-b016-f0e996cdd5b2","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:22.034Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"c3846c83-886f-4bf1-b016-f0e996cdd5b2","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:22.034Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"c3846c83-886f-4bf1-b016-f0e996cdd5b2"}
{"level":"info","ts":"2026-07-13T15:50:22.034Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"c3846c83-886f-4bf1-b016-f0e996cdd5b2","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:24.987Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"38d54deb-a8b4-4659-8e75-38df76fe2a7b"}
{"level":"info","ts":"2026-07-13T15:50:24.987Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"38d54deb-a8b4-4659-8e75-38df76fe2a7b"}
{"level":"info","ts":"2026-07-13T15:50:24.987Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"38d54deb-a8b4-4659-8e75-38df76fe2a7b","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:24.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"38d54deb-a8b4-4659-8e75-38df76fe2a7b","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:50:24.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"38d54deb-a8b4-4659-8e75-38df76fe2a7b","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:24.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"38d54deb-a8b4-4659-8e75-38df76fe2a7b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:50:24.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"38d54deb-a8b4-4659-8e75-38df76fe2a7b","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:24.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"38d54deb-a8b4-4659-8e75-38df76fe2a7b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:24.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"38d54deb-a8b4-4659-8e75-38df76fe2a7b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:50:24.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"38d54deb-a8b4-4659-8e75-38df76fe2a7b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:24.988Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"38d54deb-a8b4-4659-8e75-38df76fe2a7b","podName":"fail-pbfxx-small-group-worker-bhr2g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:24.988Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"38d54deb-a8b4-4659-8e75-38df76fe2a7b","podName":"fail-pbfxx-small-group-worker-bhr2g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:24.988Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"38d54deb-a8b4-4659-8e75-38df76fe2a7b","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:50:24Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:50:24.988Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"38d54deb-a8b4-4659-8e75-38df76fe2a7b","name":"fail-pbfxx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:50:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:50:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:50:24Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:50:25.001Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"fa0cc12a-6d12-45c9-8334-55ca6820cb11","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:25.001Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"c5b78140-ea1c-4564-a1c4-96212c691a58","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:25.001Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"c5b78140-ea1c-4564-a1c4-96212c691a58","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:25.001Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"c5b78140-ea1c-4564-a1c4-96212c691a58","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:25.001Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"d4303dfb-2974-49a7-af6d-5f2e7cced63f","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:25.001Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"fa0cc12a-6d12-45c9-8334-55ca6820cb11","pod":"fail-pbfxx-head-27cb9","ip":"10.133.0.20"}
{"level":"info","ts":"2026-07-13T15:50:25.001Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"fa0cc12a-6d12-45c9-8334-55ca6820cb11","pod":"fail-pbfxx-small-group-worker-bhr2g","ip":"10.134.0.26"}
{"level":"info","ts":"2026-07-13T15:50:25.001Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"fa0cc12a-6d12-45c9-8334-55ca6820cb11","rayCluster":"fail-pbfxx","podIPs":["10.133.0.20","10.134.0.26"]}
{"level":"info","ts":"2026-07-13T15:50:25.001Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"c5b78140-ea1c-4564-a1c4-96212c691a58"}
{"level":"info","ts":"2026-07-13T15:50:25.001Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"c5b78140-ea1c-4564-a1c4-96212c691a58","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:25.011Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"d4303dfb-2974-49a7-af6d-5f2e7cced63f","name":"fail-pbfxx-head"}
{"level":"info","ts":"2026-07-13T15:50:25.022Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"d4303dfb-2974-49a7-af6d-5f2e7cced63f","name":"fail-pbfxx-workers"}
{"level":"info","ts":"2026-07-13T15:50:25.022Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"d4303dfb-2974-49a7-af6d-5f2e7cced63f","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:25.035Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"27040878-364a-4768-99d9-55b5c51fe4fe","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:25.035Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"27040878-364a-4768-99d9-55b5c51fe4fe","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:25.035Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"27040878-364a-4768-99d9-55b5c51fe4fe","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:25.035Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"27040878-364a-4768-99d9-55b5c51fe4fe"}
{"level":"info","ts":"2026-07-13T15:50:25.035Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"27040878-364a-4768-99d9-55b5c51fe4fe","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:27.001Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"fa8d52ae-299d-46d7-8270-3d4d583a3015"}
{"level":"info","ts":"2026-07-13T15:50:27.001Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"fa8d52ae-299d-46d7-8270-3d4d583a3015"}
{"level":"info","ts":"2026-07-13T15:50:27.001Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"fa8d52ae-299d-46d7-8270-3d4d583a3015","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:50:27.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"fa8d52ae-299d-46d7-8270-3d4d583a3015","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:50:27.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"fa8d52ae-299d-46d7-8270-3d4d583a3015","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:27.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"fa8d52ae-299d-46d7-8270-3d4d583a3015","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:50:27.002Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"fa8d52ae-299d-46d7-8270-3d4d583a3015","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:50:27.002Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"fa8d52ae-299d-46d7-8270-3d4d583a3015","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:27.002Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"fa8d52ae-299d-46d7-8270-3d4d583a3015","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:50:27.002Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"fa8d52ae-299d-46d7-8270-3d4d583a3015","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:50:27.002Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"fa8d52ae-299d-46d7-8270-3d4d583a3015","podName":"fail-pbfxx-small-group-worker-bhr2g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:27.002Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"fa8d52ae-299d-46d7-8270-3d4d583a3015","podName":"fail-pbfxx-small-group-worker-bhr2g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:50:27.002Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"fa8d52ae-299d-46d7-8270-3d4d583a3015","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:50:27.002Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"fa8d52ae-299d-46d7-8270-3d4d583a3015","seconds":300}
{"level":"info","ts":"2026-07-13T15:50:28.035Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"5e304169-4ec2-4151-835b-cfa4dc2c16be","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:28.035Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"5e304169-4ec2-4151-835b-cfa4dc2c16be","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:28.035Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"5e304169-4ec2-4151-835b-cfa4dc2c16be","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:28.036Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"5e304169-4ec2-4151-835b-cfa4dc2c16be"}
{"level":"info","ts":"2026-07-13T15:50:28.036Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"5e304169-4ec2-4151-835b-cfa4dc2c16be","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:31.036Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"a35cd568-9d6e-4cdd-a02a-ffc88da7848d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:31.036Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"a35cd568-9d6e-4cdd-a02a-ffc88da7848d","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:31.037Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"a35cd568-9d6e-4cdd-a02a-ffc88da7848d","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:31.037Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"a35cd568-9d6e-4cdd-a02a-ffc88da7848d"}
{"level":"info","ts":"2026-07-13T15:50:31.037Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"a35cd568-9d6e-4cdd-a02a-ffc88da7848d","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:34.037Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"3b659510-5288-47f1-83c0-37f8b8ee5688","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:34.037Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"3b659510-5288-47f1-83c0-37f8b8ee5688","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:34.037Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"3b659510-5288-47f1-83c0-37f8b8ee5688","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:34.038Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"3b659510-5288-47f1-83c0-37f8b8ee5688"}
{"level":"info","ts":"2026-07-13T15:50:34.038Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"3b659510-5288-47f1-83c0-37f8b8ee5688","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:34.222Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"1b1136f6-97d9-4ea2-a226-e34deb9d36a4","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:34.222Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"1b1136f6-97d9-4ea2-a226-e34deb9d36a4","pod":"fail-pbfxx-head-27cb9","ip":"10.133.0.20"}
{"level":"info","ts":"2026-07-13T15:50:34.222Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"1b1136f6-97d9-4ea2-a226-e34deb9d36a4","pod":"fail-pbfxx-small-group-worker-bhr2g","ip":"10.134.0.26"}
{"level":"info","ts":"2026-07-13T15:50:34.222Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"1b1136f6-97d9-4ea2-a226-e34deb9d36a4","rayCluster":"fail-pbfxx","podIPs":["10.133.0.20","10.134.0.26"]}
{"level":"info","ts":"2026-07-13T15:50:34.934Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"03b26b5f-be4a-478c-a8ef-6b64ceed32de","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:50:34.934Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"03b26b5f-be4a-478c-a8ef-6b64ceed32de","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T15:50:34.934Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"03b26b5f-be4a-478c-a8ef-6b64ceed32de","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T15:50:34.934Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"03b26b5f-be4a-478c-a8ef-6b64ceed32de","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T15:50:37.038Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"88e167c2-94a5-4fc2-9197-7bd602ea8d46","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:37.038Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"88e167c2-94a5-4fc2-9197-7bd602ea8d46","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:37.038Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"88e167c2-94a5-4fc2-9197-7bd602ea8d46","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:37.039Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"88e167c2-94a5-4fc2-9197-7bd602ea8d46"}
{"level":"info","ts":"2026-07-13T15:50:37.039Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"88e167c2-94a5-4fc2-9197-7bd602ea8d46","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:40.040Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"518cef24-5dff-4640-9b0a-33b7b29d3f57","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:40.040Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"518cef24-5dff-4640-9b0a-33b7b29d3f57","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:40.040Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"518cef24-5dff-4640-9b0a-33b7b29d3f57","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:40.040Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"518cef24-5dff-4640-9b0a-33b7b29d3f57"}
{"level":"info","ts":"2026-07-13T15:50:40.040Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"518cef24-5dff-4640-9b0a-33b7b29d3f57","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:43.041Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"70eac065-0f60-4728-b047-bbf887e4cc22","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:43.041Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"70eac065-0f60-4728-b047-bbf887e4cc22","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:43.041Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"70eac065-0f60-4728-b047-bbf887e4cc22","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:43.041Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"70eac065-0f60-4728-b047-bbf887e4cc22"}
{"level":"info","ts":"2026-07-13T15:50:43.041Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"70eac065-0f60-4728-b047-bbf887e4cc22","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:46.042Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"536bb521-5fcf-4fac-9cb2-7617d904c48b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:46.042Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"536bb521-5fcf-4fac-9cb2-7617d904c48b","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:46.042Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"536bb521-5fcf-4fac-9cb2-7617d904c48b","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:46.042Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"536bb521-5fcf-4fac-9cb2-7617d904c48b"}
{"level":"info","ts":"2026-07-13T15:50:46.042Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"536bb521-5fcf-4fac-9cb2-7617d904c48b","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:49.042Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"a6ea02b6-9bfb-4e7d-8580-a6b5ce31fde8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:49.042Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"a6ea02b6-9bfb-4e7d-8580-a6b5ce31fde8","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:49.042Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"a6ea02b6-9bfb-4e7d-8580-a6b5ce31fde8","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:49.043Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"a6ea02b6-9bfb-4e7d-8580-a6b5ce31fde8"}
{"level":"info","ts":"2026-07-13T15:50:49.043Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"a6ea02b6-9bfb-4e7d-8580-a6b5ce31fde8","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:52.043Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"824a427c-0601-48f7-bfa8-92daf3e8b746","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:52.043Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"824a427c-0601-48f7-bfa8-92daf3e8b746","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:52.043Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"824a427c-0601-48f7-bfa8-92daf3e8b746","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:52.043Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"824a427c-0601-48f7-bfa8-92daf3e8b746"}
{"level":"info","ts":"2026-07-13T15:50:52.043Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"824a427c-0601-48f7-bfa8-92daf3e8b746","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:55.043Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"af6a8bfe-7b63-4db1-8c98-910b6be1f8d4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:55.043Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"af6a8bfe-7b63-4db1-8c98-910b6be1f8d4","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:55.043Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"af6a8bfe-7b63-4db1-8c98-910b6be1f8d4","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:55.043Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"af6a8bfe-7b63-4db1-8c98-910b6be1f8d4"}
{"level":"info","ts":"2026-07-13T15:50:55.043Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"af6a8bfe-7b63-4db1-8c98-910b6be1f8d4","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:50:58.044Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"55247e04-f54f-4a12-b876-241aad3bdc7d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:50:58.044Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"55247e04-f54f-4a12-b876-241aad3bdc7d","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:50:58.044Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"55247e04-f54f-4a12-b876-241aad3bdc7d","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:50:58.044Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"55247e04-f54f-4a12-b876-241aad3bdc7d"}
{"level":"info","ts":"2026-07-13T15:50:58.044Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"55247e04-f54f-4a12-b876-241aad3bdc7d","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:51:01.044Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"053ce7e2-2f7e-47e2-84dd-b3a32f617dfd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:01.044Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"053ce7e2-2f7e-47e2-84dd-b3a32f617dfd","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:01.044Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"053ce7e2-2f7e-47e2-84dd-b3a32f617dfd","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:01.045Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"053ce7e2-2f7e-47e2-84dd-b3a32f617dfd"}
{"level":"info","ts":"2026-07-13T15:51:01.045Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"053ce7e2-2f7e-47e2-84dd-b3a32f617dfd","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:51:04.045Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"71a0ab17-5c7e-42a6-8b71-caf75591c6e8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:04.045Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"71a0ab17-5c7e-42a6-8b71-caf75591c6e8","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:04.045Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"71a0ab17-5c7e-42a6-8b71-caf75591c6e8","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:04.046Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"71a0ab17-5c7e-42a6-8b71-caf75591c6e8"}
{"level":"info","ts":"2026-07-13T15:51:04.046Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"71a0ab17-5c7e-42a6-8b71-caf75591c6e8","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:51:07.047Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"a69710e5-22d0-4977-85e2-349a3595b7f5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:07.047Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"a69710e5-22d0-4977-85e2-349a3595b7f5","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:07.047Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"a69710e5-22d0-4977-85e2-349a3595b7f5","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:07.047Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"a69710e5-22d0-4977-85e2-349a3595b7f5"}
{"level":"info","ts":"2026-07-13T15:51:07.047Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"a69710e5-22d0-4977-85e2-349a3595b7f5","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:51:10.047Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"9f191212-a0ec-4437-8406-c8b68432ba8a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:10.048Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"9f191212-a0ec-4437-8406-c8b68432ba8a","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:10.048Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"9f191212-a0ec-4437-8406-c8b68432ba8a","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:10.048Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"9f191212-a0ec-4437-8406-c8b68432ba8a"}
{"level":"info","ts":"2026-07-13T15:51:10.048Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"9f191212-a0ec-4437-8406-c8b68432ba8a","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:51:13.048Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"d3505e38-5037-4ce1-9c49-26abbc6c8155","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:13.048Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"d3505e38-5037-4ce1-9c49-26abbc6c8155","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:13.048Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"d3505e38-5037-4ce1-9c49-26abbc6c8155","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:13.049Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"d3505e38-5037-4ce1-9c49-26abbc6c8155"}
{"level":"info","ts":"2026-07-13T15:51:13.049Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"d3505e38-5037-4ce1-9c49-26abbc6c8155","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:51:13.270Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6b15dd92-6916-4f7d-ae6c-79e57ca14b53"}
{"level":"info","ts":"2026-07-13T15:51:13.270Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6b15dd92-6916-4f7d-ae6c-79e57ca14b53"}
{"level":"info","ts":"2026-07-13T15:51:13.270Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6b15dd92-6916-4f7d-ae6c-79e57ca14b53","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:13.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6b15dd92-6916-4f7d-ae6c-79e57ca14b53","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:51:13.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6b15dd92-6916-4f7d-ae6c-79e57ca14b53","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:13.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6b15dd92-6916-4f7d-ae6c-79e57ca14b53","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:51:13.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6b15dd92-6916-4f7d-ae6c-79e57ca14b53","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:13.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6b15dd92-6916-4f7d-ae6c-79e57ca14b53","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:13.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6b15dd92-6916-4f7d-ae6c-79e57ca14b53","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:51:13.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6b15dd92-6916-4f7d-ae6c-79e57ca14b53","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:13.271Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6b15dd92-6916-4f7d-ae6c-79e57ca14b53","podName":"fail-pbfxx-small-group-worker-bhr2g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:13.271Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6b15dd92-6916-4f7d-ae6c-79e57ca14b53","podName":"fail-pbfxx-small-group-worker-bhr2g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:13.271Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6b15dd92-6916-4f7d-ae6c-79e57ca14b53","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:51:13.271Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6b15dd92-6916-4f7d-ae6c-79e57ca14b53","seconds":300}
{"level":"info","ts":"2026-07-13T15:51:14.279Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3d1f0a35-598c-436d-bb47-382d7a4a36c8"}
{"level":"info","ts":"2026-07-13T15:51:14.279Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3d1f0a35-598c-436d-bb47-382d7a4a36c8"}
{"level":"info","ts":"2026-07-13T15:51:14.279Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3d1f0a35-598c-436d-bb47-382d7a4a36c8","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:14.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3d1f0a35-598c-436d-bb47-382d7a4a36c8","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:51:14.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3d1f0a35-598c-436d-bb47-382d7a4a36c8","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:14.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3d1f0a35-598c-436d-bb47-382d7a4a36c8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:51:14.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3d1f0a35-598c-436d-bb47-382d7a4a36c8","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:14.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3d1f0a35-598c-436d-bb47-382d7a4a36c8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:14.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3d1f0a35-598c-436d-bb47-382d7a4a36c8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:51:14.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3d1f0a35-598c-436d-bb47-382d7a4a36c8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:14.280Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3d1f0a35-598c-436d-bb47-382d7a4a36c8","podName":"fail-pbfxx-small-group-worker-bhr2g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:14.280Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3d1f0a35-598c-436d-bb47-382d7a4a36c8","podName":"fail-pbfxx-small-group-worker-bhr2g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:14.280Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3d1f0a35-598c-436d-bb47-382d7a4a36c8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:51:14.280Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3d1f0a35-598c-436d-bb47-382d7a4a36c8","seconds":300}
{"level":"info","ts":"2026-07-13T15:51:16.049Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"330ac9ce-0a6e-476f-8dff-70d1bbd860a9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:16.049Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"330ac9ce-0a6e-476f-8dff-70d1bbd860a9","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:16.049Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"330ac9ce-0a6e-476f-8dff-70d1bbd860a9","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:16.049Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"330ac9ce-0a6e-476f-8dff-70d1bbd860a9"}
{"level":"info","ts":"2026-07-13T15:51:16.049Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"330ac9ce-0a6e-476f-8dff-70d1bbd860a9","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:51:19.050Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"2bacd934-ea15-4830-95e4-4e8a41ee960f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:19.050Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"2bacd934-ea15-4830-95e4-4e8a41ee960f","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:19.050Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"2bacd934-ea15-4830-95e4-4e8a41ee960f","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:19.050Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"2bacd934-ea15-4830-95e4-4e8a41ee960f"}
{"level":"info","ts":"2026-07-13T15:51:19.050Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"2bacd934-ea15-4830-95e4-4e8a41ee960f","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:51:22.051Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"5ad63731-7203-4010-b523-14c553afffbe","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:22.051Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"5ad63731-7203-4010-b523-14c553afffbe","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:22.051Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"5ad63731-7203-4010-b523-14c553afffbe","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:22.051Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"5ad63731-7203-4010-b523-14c553afffbe"}
{"level":"info","ts":"2026-07-13T15:51:22.051Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"5ad63731-7203-4010-b523-14c553afffbe","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:51:24.319Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"986fabb8-d386-49ba-9e50-2772f12e4f1e"}
{"level":"info","ts":"2026-07-13T15:51:24.319Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"986fabb8-d386-49ba-9e50-2772f12e4f1e"}
{"level":"info","ts":"2026-07-13T15:51:24.319Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"986fabb8-d386-49ba-9e50-2772f12e4f1e","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:24.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"986fabb8-d386-49ba-9e50-2772f12e4f1e","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:51:24.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"986fabb8-d386-49ba-9e50-2772f12e4f1e","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:24.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"986fabb8-d386-49ba-9e50-2772f12e4f1e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:51:24.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"986fabb8-d386-49ba-9e50-2772f12e4f1e","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:24.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"986fabb8-d386-49ba-9e50-2772f12e4f1e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:24.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"986fabb8-d386-49ba-9e50-2772f12e4f1e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:51:24.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"986fabb8-d386-49ba-9e50-2772f12e4f1e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:24.319Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"986fabb8-d386-49ba-9e50-2772f12e4f1e","podName":"fail-pbfxx-small-group-worker-bhr2g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:24.319Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"986fabb8-d386-49ba-9e50-2772f12e4f1e","podName":"fail-pbfxx-small-group-worker-bhr2g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:24.319Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"986fabb8-d386-49ba-9e50-2772f12e4f1e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:51:24.319Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"986fabb8-d386-49ba-9e50-2772f12e4f1e","seconds":300}
{"level":"info","ts":"2026-07-13T15:51:25.051Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"34ac9ae9-3a53-4fd7-b7ab-1bc5d59abe97","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:25.051Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"34ac9ae9-3a53-4fd7-b7ab-1bc5d59abe97","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:25.051Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"34ac9ae9-3a53-4fd7-b7ab-1bc5d59abe97","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:25.052Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"34ac9ae9-3a53-4fd7-b7ab-1bc5d59abe97"}
{"level":"info","ts":"2026-07-13T15:51:25.052Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"34ac9ae9-3a53-4fd7-b7ab-1bc5d59abe97","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:51:25.323Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"93a21b92-8471-4580-a709-a97862715cfe"}
{"level":"info","ts":"2026-07-13T15:51:25.323Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"93a21b92-8471-4580-a709-a97862715cfe"}
{"level":"info","ts":"2026-07-13T15:51:25.323Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"93a21b92-8471-4580-a709-a97862715cfe","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:25.323Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"93a21b92-8471-4580-a709-a97862715cfe","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:51:25.323Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"93a21b92-8471-4580-a709-a97862715cfe","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:25.323Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"93a21b92-8471-4580-a709-a97862715cfe","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:51:25.323Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"93a21b92-8471-4580-a709-a97862715cfe","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:25.323Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"93a21b92-8471-4580-a709-a97862715cfe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:25.323Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"93a21b92-8471-4580-a709-a97862715cfe","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:51:25.323Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"93a21b92-8471-4580-a709-a97862715cfe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:25.323Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"93a21b92-8471-4580-a709-a97862715cfe","podName":"fail-pbfxx-small-group-worker-bhr2g","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:50:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:51:25.323Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"93a21b92-8471-4580-a709-a97862715cfe","podName":"fail-pbfxx-small-group-worker-bhr2g","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:50:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:51:25.323Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"93a21b92-8471-4580-a709-a97862715cfe","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:51:25.323Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"93a21b92-8471-4580-a709-a97862715cfe","name":"fail-pbfxx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:50:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:50:24Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:50:24Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:51:25.335Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"1afe57e7-cf18-445d-8e78-155247dc35ce","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:25.335Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"1048c7a3-2b21-4273-9285-623a8ea76092","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:25.335Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"7a6a76c4-42c9-4e24-88b7-9c3dbc8fcb72","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:25.335Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"7a6a76c4-42c9-4e24-88b7-9c3dbc8fcb72","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:25.335Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"1048c7a3-2b21-4273-9285-623a8ea76092","pod":"fail-pbfxx-head-27cb9","ip":"10.133.0.20"}
{"level":"info","ts":"2026-07-13T15:51:25.335Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"1048c7a3-2b21-4273-9285-623a8ea76092","pod":"fail-pbfxx-small-group-worker-bhr2g","ip":"10.134.0.26"}
{"level":"info","ts":"2026-07-13T15:51:25.335Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"7a6a76c4-42c9-4e24-88b7-9c3dbc8fcb72","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:25.335Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"1048c7a3-2b21-4273-9285-623a8ea76092","rayCluster":"fail-pbfxx","podIPs":["10.133.0.20","10.134.0.26"]}
{"level":"info","ts":"2026-07-13T15:51:25.335Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"7a6a76c4-42c9-4e24-88b7-9c3dbc8fcb72"}
{"level":"info","ts":"2026-07-13T15:51:25.335Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"7a6a76c4-42c9-4e24-88b7-9c3dbc8fcb72","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:51:25.352Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"1afe57e7-cf18-445d-8e78-155247dc35ce","name":"fail-pbfxx-head"}
{"level":"info","ts":"2026-07-13T15:51:25.363Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"1afe57e7-cf18-445d-8e78-155247dc35ce","name":"fail-pbfxx-workers"}
{"level":"info","ts":"2026-07-13T15:51:25.363Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"1afe57e7-cf18-445d-8e78-155247dc35ce","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:27.335Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6a02cba7-f53b-4b1e-9f01-eea9ef39751c"}
{"level":"info","ts":"2026-07-13T15:51:27.335Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6a02cba7-f53b-4b1e-9f01-eea9ef39751c"}
{"level":"info","ts":"2026-07-13T15:51:27.335Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6a02cba7-f53b-4b1e-9f01-eea9ef39751c","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:27.335Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6a02cba7-f53b-4b1e-9f01-eea9ef39751c","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:51:27.335Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6a02cba7-f53b-4b1e-9f01-eea9ef39751c","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:27.335Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6a02cba7-f53b-4b1e-9f01-eea9ef39751c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:51:27.335Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6a02cba7-f53b-4b1e-9f01-eea9ef39751c","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:27.335Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6a02cba7-f53b-4b1e-9f01-eea9ef39751c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:27.335Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6a02cba7-f53b-4b1e-9f01-eea9ef39751c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:51:27.335Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6a02cba7-f53b-4b1e-9f01-eea9ef39751c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:27.335Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6a02cba7-f53b-4b1e-9f01-eea9ef39751c","podName":"fail-pbfxx-small-group-worker-bhr2g","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:50:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:51:27.335Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6a02cba7-f53b-4b1e-9f01-eea9ef39751c","podName":"fail-pbfxx-small-group-worker-bhr2g","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:50:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:51:27.335Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6a02cba7-f53b-4b1e-9f01-eea9ef39751c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:51:27.335Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"6a02cba7-f53b-4b1e-9f01-eea9ef39751c","seconds":300}
{"level":"info","ts":"2026-07-13T15:51:28.052Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"c31a2264-b6be-4363-8f84-79af2178f280","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:28.052Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"c31a2264-b6be-4363-8f84-79af2178f280","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:28.052Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"c31a2264-b6be-4363-8f84-79af2178f280","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:28.052Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"c31a2264-b6be-4363-8f84-79af2178f280"}
{"level":"info","ts":"2026-07-13T15:51:28.052Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"c31a2264-b6be-4363-8f84-79af2178f280","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:51:31.054Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"e43d121e-89b7-412c-9e6e-5cbf29507bc3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:31.054Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"e43d121e-89b7-412c-9e6e-5cbf29507bc3","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:31.054Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"e43d121e-89b7-412c-9e6e-5cbf29507bc3","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:31.054Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"e43d121e-89b7-412c-9e6e-5cbf29507bc3"}
{"level":"info","ts":"2026-07-13T15:51:31.054Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"e43d121e-89b7-412c-9e6e-5cbf29507bc3","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:51:34.054Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"2d235212-e94a-4b5c-a7dc-428244e7e75d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:34.054Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"2d235212-e94a-4b5c-a7dc-428244e7e75d","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:34.054Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"2d235212-e94a-4b5c-a7dc-428244e7e75d","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:34.055Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"2d235212-e94a-4b5c-a7dc-428244e7e75d"}
{"level":"info","ts":"2026-07-13T15:51:34.055Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"2d235212-e94a-4b5c-a7dc-428244e7e75d","RayCluster":"fail-pbfxx","State":""}
{"level":"info","ts":"2026-07-13T15:51:34.223Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"4b9f7924-ca31-48cc-b602-740cf519a9d4","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:34.223Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"4b9f7924-ca31-48cc-b602-740cf519a9d4","pod":"fail-pbfxx-head-27cb9","ip":"10.133.0.20"}
{"level":"info","ts":"2026-07-13T15:51:34.223Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"4b9f7924-ca31-48cc-b602-740cf519a9d4","pod":"fail-pbfxx-small-group-worker-bhr2g","ip":"10.134.0.26"}
{"level":"info","ts":"2026-07-13T15:51:34.223Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"4b9f7924-ca31-48cc-b602-740cf519a9d4","rayCluster":"fail-pbfxx","podIPs":["10.133.0.20","10.134.0.26"]}
{"level":"info","ts":"2026-07-13T15:51:34.935Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8477f61b-b4b0-4dcf-b9f5-63fd77117b47","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:51:34.935Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8477f61b-b4b0-4dcf-b9f5-63fd77117b47","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T15:51:34.935Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8477f61b-b4b0-4dcf-b9f5-63fd77117b47","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T15:51:34.935Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8477f61b-b4b0-4dcf-b9f5-63fd77117b47","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T15:51:36.351Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"9b7e9159-3ae0-4b1c-b88d-9d782f1b0d69"}
{"level":"info","ts":"2026-07-13T15:51:36.351Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"9b7e9159-3ae0-4b1c-b88d-9d782f1b0d69"}
{"level":"info","ts":"2026-07-13T15:51:36.351Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"9b7e9159-3ae0-4b1c-b88d-9d782f1b0d69","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:36.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"9b7e9159-3ae0-4b1c-b88d-9d782f1b0d69","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:51:36.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"9b7e9159-3ae0-4b1c-b88d-9d782f1b0d69","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:36.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"9b7e9159-3ae0-4b1c-b88d-9d782f1b0d69","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:51:36.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"9b7e9159-3ae0-4b1c-b88d-9d782f1b0d69","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:36.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"9b7e9159-3ae0-4b1c-b88d-9d782f1b0d69","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:36.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"9b7e9159-3ae0-4b1c-b88d-9d782f1b0d69","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:51:36.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"9b7e9159-3ae0-4b1c-b88d-9d782f1b0d69","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:36.351Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"9b7e9159-3ae0-4b1c-b88d-9d782f1b0d69","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-13T15:51:36.351Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"9b7e9159-3ae0-4b1c-b88d-9d782f1b0d69","name":"fail-pbfxx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:50:24Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:36Z","stateTransitionTimes":{"ready":"2026-07-13T15:51:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:50:24Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:51:36.363Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"2e895456-120a-4dab-9b94-81da06933bf2","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:36.363Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"63648997-2f92-490b-b418-b5ae051cdfe3","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:36.363Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"2e895456-120a-4dab-9b94-81da06933bf2","pod":"fail-pbfxx-small-group-worker-bhr2g","ip":"10.134.0.26"}
{"level":"info","ts":"2026-07-13T15:51:36.363Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"2e895456-120a-4dab-9b94-81da06933bf2","pod":"fail-pbfxx-head-27cb9","ip":"10.133.0.20"}
{"level":"info","ts":"2026-07-13T15:51:36.363Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"2e895456-120a-4dab-9b94-81da06933bf2","rayCluster":"fail-pbfxx","podIPs":["10.134.0.26","10.133.0.20"]}
{"level":"info","ts":"2026-07-13T15:51:36.363Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"b25a52c4-c8d4-4d40-a003-cd33aa1802d8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:36.363Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"b25a52c4-c8d4-4d40-a003-cd33aa1802d8","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:36.363Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"b25a52c4-c8d4-4d40-a003-cd33aa1802d8","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:36.363Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"b25a52c4-c8d4-4d40-a003-cd33aa1802d8"}
{"level":"info","ts":"2026-07-13T15:51:36.363Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"b25a52c4-c8d4-4d40-a003-cd33aa1802d8","head service name":"fail-pbfxx-head-svc","namespace":"test-ns-5xvz7"}
{"level":"info","ts":"2026-07-13T15:51:36.363Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"b25a52c4-c8d4-4d40-a003-cd33aa1802d8","head service URL":"fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-13T15:51:36.363Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"b25a52c4-c8d4-4d40-a003-cd33aa1802d8","SubmissionMode":"HTTPMode","RayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:36.363Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"b25a52c4-c8d4-4d40-a003-cd33aa1802d8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-kh6fm","rayClusterName":"fail-pbfxx","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:50:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-kh6fm","rayClusterName":"fail-pbfxx","dashboardURL":"fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T15:50:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:51:36.363Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"b25a52c4-c8d4-4d40-a003-cd33aa1802d8","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T15:51:36.372Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"63648997-2f92-490b-b418-b5ae051cdfe3","name":"fail-pbfxx-head"}
{"level":"info","ts":"2026-07-13T15:51:36.373Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"4e015415-200b-4531-9113-617436d82545","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:36.373Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"4e015415-200b-4531-9113-617436d82545","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:36.373Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"4e015415-200b-4531-9113-617436d82545","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:36.373Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"4e015415-200b-4531-9113-617436d82545"}
{"level":"info","ts":"2026-07-13T15:51:36.381Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"4e015415-200b-4531-9113-617436d82545","JobId":"fail-kh6fm"}
{"level":"info","ts":"2026-07-13T15:51:36.381Z","logger":"controllers.RayJob","msg":"Submit a ray job","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"4e015415-200b-4531-9113-617436d82545","rayJob":"fail","jobInfo":"{\"entrypoint\":\"python /home/ray/jobs/fail.py\",\"submission_id\":\"fail-kh6fm\"}"}
{"level":"info","ts":"2026-07-13T15:51:36.383Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"63648997-2f92-490b-b418-b5ae051cdfe3","name":"fail-pbfxx-workers"}
{"level":"info","ts":"2026-07-13T15:51:36.383Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"63648997-2f92-490b-b418-b5ae051cdfe3","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:37.264Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"32349c74-dc6d-495f-83e3-c2bc1f7b8901","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:37.264Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"32349c74-dc6d-495f-83e3-c2bc1f7b8901","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:37.264Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"32349c74-dc6d-495f-83e3-c2bc1f7b8901","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:37.264Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"32349c74-dc6d-495f-83e3-c2bc1f7b8901"}
{"level":"info","ts":"2026-07-13T15:51:37.268Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"32349c74-dc6d-495f-83e3-c2bc1f7b8901","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-kh6fm","rayClusterName":"fail-pbfxx","dashboardURL":"fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T15:50:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:51:36Z"},"jobId":"fail-kh6fm","rayClusterName":"fail-pbfxx","dashboardURL":"fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-13T15:50:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:36Z","stateTransitionTimes":{"ready":"2026-07-13T15:51:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:50:24Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:51:37.268Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"32349c74-dc6d-495f-83e3-c2bc1f7b8901","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T15:51:37.280Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"0093c57b-29be-4191-b41d-6b27f1dcfa35","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:37.280Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"0093c57b-29be-4191-b41d-6b27f1dcfa35","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:37.280Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"0093c57b-29be-4191-b41d-6b27f1dcfa35","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:37.281Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"0093c57b-29be-4191-b41d-6b27f1dcfa35"}
{"level":"info","ts":"2026-07-13T15:51:37.283Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"0093c57b-29be-4191-b41d-6b27f1dcfa35","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:51:36Z"},"jobId":"fail-kh6fm","rayClusterName":"fail-pbfxx","dashboardURL":"fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-13T15:50:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:36Z","stateTransitionTimes":{"ready":"2026-07-13T15:51:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:50:24Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:51:36Z"},"jobId":"fail-kh6fm","rayClusterName":"fail-pbfxx","dashboardURL":"fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-13T15:50:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:36Z","stateTransitionTimes":{"ready":"2026-07-13T15:51:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:50:24Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:51:38.363Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c781a256-7ca0-4e65-906e-4f7ca6690a82"}
{"level":"info","ts":"2026-07-13T15:51:38.363Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c781a256-7ca0-4e65-906e-4f7ca6690a82"}
{"level":"info","ts":"2026-07-13T15:51:38.363Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c781a256-7ca0-4e65-906e-4f7ca6690a82","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:38.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c781a256-7ca0-4e65-906e-4f7ca6690a82","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:51:38.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c781a256-7ca0-4e65-906e-4f7ca6690a82","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:38.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c781a256-7ca0-4e65-906e-4f7ca6690a82","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:51:38.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c781a256-7ca0-4e65-906e-4f7ca6690a82","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:38.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c781a256-7ca0-4e65-906e-4f7ca6690a82","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:38.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c781a256-7ca0-4e65-906e-4f7ca6690a82","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:51:38.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c781a256-7ca0-4e65-906e-4f7ca6690a82","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:38.364Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c781a256-7ca0-4e65-906e-4f7ca6690a82","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:51:38.364Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c781a256-7ca0-4e65-906e-4f7ca6690a82","seconds":300}
{"level":"info","ts":"2026-07-13T15:51:40.265Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"ead907e2-0b7d-4088-a9d7-23ecc67c0727","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:40.265Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"ead907e2-0b7d-4088-a9d7-23ecc67c0727","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:51:40.265Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"ead907e2-0b7d-4088-a9d7-23ecc67c0727","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:40.265Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"ead907e2-0b7d-4088-a9d7-23ecc67c0727"}
{"level":"info","ts":"2026-07-13T15:51:40.268Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"ead907e2-0b7d-4088-a9d7-23ecc67c0727","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:51:36Z"},"jobId":"fail-kh6fm","rayClusterName":"fail-pbfxx","dashboardURL":"fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-13T15:50:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:36Z","stateTransitionTimes":{"ready":"2026-07-13T15:51:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:50:24Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:51:36Z","endTime":"2026-07-13T15:51:38Z"},"jobId":"fail-kh6fm","rayClusterName":"fail-pbfxx","dashboardURL":"fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Failed","reason":"AppFailed","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 15:51:36,932\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-kh6fm: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T15:50:03Z","succeeded":0,"failed":1,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:36Z","stateTransitionTimes":{"ready":"2026-07-13T15:51:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:50:24Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:51:40.268Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"ead907e2-0b7d-4088-a9d7-23ecc67c0727","old JobStatus":"PENDING","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-07-13T15:51:40.281Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"534d323c-af0d-48f4-bf36-38cf71a93d0f","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:40.281Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"534d323c-af0d-48f4-bf36-38cf71a93d0f","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 15:51:40 +0000 UTC","Now":"2026-07-13T15:51:40.281Z","ShutdownTime":"2026-07-13T15:51:40.000Z"}
{"level":"info","ts":"2026-07-13T15:51:40.421Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"06098498-b295-4e07-a74d-7b4a52fe6da2","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:40.421Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"06098498-b295-4e07-a74d-7b4a52fe6da2","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T15:51:40.431Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"06098498-b295-4e07-a74d-7b4a52fe6da2"}
{"level":"info","ts":"2026-07-13T15:51:40.431Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"06098498-b295-4e07-a74d-7b4a52fe6da2","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-13T15:51:40.431Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"06098498-b295-4e07-a74d-7b4a52fe6da2","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:51:40.431Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"06098498-b295-4e07-a74d-7b4a52fe6da2","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T15:51:40.441Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"bb56a87d-e633-45f0-864d-4ebf2d3005ef","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:40.441Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"bb56a87d-e633-45f0-864d-4ebf2d3005ef"}
{"level":"info","ts":"2026-07-13T15:51:40.441Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"bb56a87d-e633-45f0-864d-4ebf2d3005ef","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-13T15:51:40.441Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"bb56a87d-e633-45f0-864d-4ebf2d3005ef","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-m7jwx","rayClusterName":"stop-h9f9m","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:51:40.441Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"bb56a87d-e633-45f0-864d-4ebf2d3005ef","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T15:51:40.446Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"bb56a87d-e633-45f0-864d-4ebf2d3005ef","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-13T15:51:40.446Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"bb56a87d-e633-45f0-864d-4ebf2d3005ef"}
{"level":"error","ts":"2026-07-13T15:51:40.446Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"bb56a87d-e633-45f0-864d-4ebf2d3005ef","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:40.446Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"95967b46-bf4b-494c-af77-86bd04f147d6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:40.446Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"95967b46-bf4b-494c-af77-86bd04f147d6","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.446Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"95967b46-bf4b-494c-af77-86bd04f147d6","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:40.446Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"95967b46-bf4b-494c-af77-86bd04f147d6","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:40.462Z","logger":"raycluster-resource","msg":"default","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.462Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-bvlzx","namespace":"test-ns-5xvz7"}
{"level":"info","ts":"2026-07-13T15:51:40.477Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"95967b46-bf4b-494c-af77-86bd04f147d6","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:40.477Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"45ea3c61-1e18-4414-907a-617a5bd9343f","namespacedName":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:40.477Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"45ea3c61-1e18-4414-907a-617a5bd9343f","mode":"IntegratedOAuth","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.477Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"45ea3c61-1e18-4414-907a-617a5bd9343f","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.477Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"45ea3c61-1e18-4414-907a-617a5bd9343f","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.477Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"95967b46-bf4b-494c-af77-86bd04f147d6"}
{"level":"info","ts":"2026-07-13T15:51:40.477Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"ab03cd7e-5ffe-4c89-b5b5-8cf421b86842","rayCluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.477Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"b47203d8-cdb2-436c-b5d3-cd787b60672d","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.477Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"95967b46-bf4b-494c-af77-86bd04f147d6","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:51:40.477Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc03920-1a24-42de-94f9-8e7cbcba34fb"}
{"level":"info","ts":"2026-07-13T15:51:40.477Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc03920-1a24-42de-94f9-8e7cbcba34fb"}
{"level":"info","ts":"2026-07-13T15:51:40.477Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"ab03cd7e-5ffe-4c89-b5b5-8cf421b86842","rayCluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.477Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"77b5bfab-6632-4485-8d3e-4f155f75cab7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:40.477Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"77b5bfab-6632-4485-8d3e-4f155f75cab7","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.477Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"77b5bfab-6632-4485-8d3e-4f155f75cab7","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:40.477Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"77b5bfab-6632-4485-8d3e-4f155f75cab7"}
{"level":"info","ts":"2026-07-13T15:51:40.477Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"77b5bfab-6632-4485-8d3e-4f155f75cab7","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:51:40.482Z","logger":"raycluster-resource","msg":"default","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.482Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-bvlzx","namespace":"test-ns-5xvz7"}
{"level":"info","ts":"2026-07-13T15:51:40.501Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc03920-1a24-42de-94f9-8e7cbcba34fb","name":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:40.501Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc03920-1a24-42de-94f9-8e7cbcba34fb","error":"head certificate secret ray-head-secret-stop-bvlzx not found"}
{"level":"error","ts":"2026-07-13T15:51:40.501Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc03920-1a24-42de-94f9-8e7cbcba34fb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:40.501Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc03920-1a24-42de-94f9-8e7cbcba34fb","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found"}
{"level":"info","ts":"2026-07-13T15:51:40.501Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc03920-1a24-42de-94f9-8e7cbcba34fb","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:40.502Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc03920-1a24-42de-94f9-8e7cbcba34fb","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:40.502Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc03920-1a24-42de-94f9-8e7cbcba34fb","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:40.502Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc03920-1a24-42de-94f9-8e7cbcba34fb","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:51:40.502Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc03920-1a24-42de-94f9-8e7cbcba34fb","name":"stop-bvlzx","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:51:40.511Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc03920-1a24-42de-94f9-8e7cbcba34fb","name":"stop-bvlzx","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-bvlzx\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-5xvz7","name":"stop-bvlzx"}}
{"level":"info","ts":"2026-07-13T15:51:40.511Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc03920-1a24-42de-94f9-8e7cbcba34fb"}
{"level":"error","ts":"2026-07-13T15:51:40.511Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ffc03920-1a24-42de-94f9-8e7cbcba34fb","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:40.511Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"af150471-7c1e-4795-abf4-00fbc8469e15"}
{"level":"info","ts":"2026-07-13T15:51:40.511Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"af150471-7c1e-4795-abf4-00fbc8469e15"}
{"level":"info","ts":"2026-07-13T15:51:40.511Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"af150471-7c1e-4795-abf4-00fbc8469e15","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:40.511Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"af150471-7c1e-4795-abf4-00fbc8469e15","error":"head certificate secret ray-head-secret-stop-bvlzx not found"}
{"level":"error","ts":"2026-07-13T15:51:40.511Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"af150471-7c1e-4795-abf4-00fbc8469e15","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:40.511Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"af150471-7c1e-4795-abf4-00fbc8469e15","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found"}
{"level":"info","ts":"2026-07-13T15:51:40.511Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"af150471-7c1e-4795-abf4-00fbc8469e15","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:40.511Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"af150471-7c1e-4795-abf4-00fbc8469e15","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:40.511Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"af150471-7c1e-4795-abf4-00fbc8469e15","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:40.511Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"af150471-7c1e-4795-abf4-00fbc8469e15","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:51:40.511Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"af150471-7c1e-4795-abf4-00fbc8469e15","name":"stop-bvlzx","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:51:40.512Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"ab03cd7e-5ffe-4c89-b5b5-8cf421b86842","rayCluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.512Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"ab03cd7e-5ffe-4c89-b5b5-8cf421b86842","rayCluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.512Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"b47203d8-cdb2-436c-b5d3-cd787b60672d","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.512Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"7acf378e-52e0-440d-9fee-f964a4e2baea","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.513Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"1e14a838-00bb-448f-a391-67a1a1665a3e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:40.513Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"1e14a838-00bb-448f-a391-67a1a1665a3e","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.513Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"1e14a838-00bb-448f-a391-67a1a1665a3e","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:40.513Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"1e14a838-00bb-448f-a391-67a1a1665a3e"}
{"level":"info","ts":"2026-07-13T15:51:40.513Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"1e14a838-00bb-448f-a391-67a1a1665a3e","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:51:40.521Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"45ea3c61-1e18-4414-907a-617a5bd9343f","name":"stop-bvlzx-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:51:40.522Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"af150471-7c1e-4795-abf4-00fbc8469e15"}
{"level":"error","ts":"2026-07-13T15:51:40.522Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"af150471-7c1e-4795-abf4-00fbc8469e15","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:40.522Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"9fccdac8-05a6-4bb7-b42d-f06697301c03"}
{"level":"info","ts":"2026-07-13T15:51:40.522Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"9fccdac8-05a6-4bb7-b42d-f06697301c03"}
{"level":"info","ts":"2026-07-13T15:51:40.522Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"9fccdac8-05a6-4bb7-b42d-f06697301c03","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:40.522Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"9fccdac8-05a6-4bb7-b42d-f06697301c03","error":"head certificate secret ray-head-secret-stop-bvlzx not found"}
{"level":"error","ts":"2026-07-13T15:51:40.522Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"9fccdac8-05a6-4bb7-b42d-f06697301c03","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:40.522Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"9fccdac8-05a6-4bb7-b42d-f06697301c03","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found"}
{"level":"info","ts":"2026-07-13T15:51:40.522Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"9fccdac8-05a6-4bb7-b42d-f06697301c03","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:40.522Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"9fccdac8-05a6-4bb7-b42d-f06697301c03","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:40.522Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"9fccdac8-05a6-4bb7-b42d-f06697301c03","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:40.522Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"9fccdac8-05a6-4bb7-b42d-f06697301c03","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:51:40.522Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"9fccdac8-05a6-4bb7-b42d-f06697301c03","name":"stop-bvlzx","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:51:40.522Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"555d368e-d720-468b-905e-ec292f4461b1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:40.522Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"555d368e-d720-468b-905e-ec292f4461b1","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.522Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"555d368e-d720-468b-905e-ec292f4461b1","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:40.522Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"555d368e-d720-468b-905e-ec292f4461b1"}
{"level":"info","ts":"2026-07-13T15:51:40.522Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"555d368e-d720-468b-905e-ec292f4461b1","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:51:40.524Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"ab03cd7e-5ffe-4c89-b5b5-8cf421b86842","rayCluster":"stop-bvlzx","secret":"ca-secret"}
{"level":"info","ts":"2026-07-13T15:51:40.524Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"ab03cd7e-5ffe-4c89-b5b5-8cf421b86842","rayCluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.525Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"7acf378e-52e0-440d-9fee-f964a4e2baea","name":"stop-bvlzx-head"}
{"level":"info","ts":"2026-07-13T15:51:40.529Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"45ea3c61-1e18-4414-907a-617a5bd9343f","name":"test-ns-5xvz7-stop-bvlzx","operation":"created"}
{"level":"info","ts":"2026-07-13T15:51:40.529Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"9fccdac8-05a6-4bb7-b42d-f06697301c03","name":"stop-bvlzx","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-bvlzx\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-5xvz7","name":"stop-bvlzx"}}
{"level":"info","ts":"2026-07-13T15:51:40.529Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"9fccdac8-05a6-4bb7-b42d-f06697301c03"}
{"level":"error","ts":"2026-07-13T15:51:40.529Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"9fccdac8-05a6-4bb7-b42d-f06697301c03","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:40.533Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"169f1613-4107-4b95-9cd8-86dd808d12d7"}
{"level":"info","ts":"2026-07-13T15:51:40.533Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"169f1613-4107-4b95-9cd8-86dd808d12d7"}
{"level":"info","ts":"2026-07-13T15:51:40.533Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"169f1613-4107-4b95-9cd8-86dd808d12d7","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:40.533Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"169f1613-4107-4b95-9cd8-86dd808d12d7","error":"head certificate secret ray-head-secret-stop-bvlzx not found"}
{"level":"error","ts":"2026-07-13T15:51:40.533Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"169f1613-4107-4b95-9cd8-86dd808d12d7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:40.533Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"169f1613-4107-4b95-9cd8-86dd808d12d7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found"}
{"level":"info","ts":"2026-07-13T15:51:40.533Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"169f1613-4107-4b95-9cd8-86dd808d12d7","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:40.533Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"169f1613-4107-4b95-9cd8-86dd808d12d7","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:40.533Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"169f1613-4107-4b95-9cd8-86dd808d12d7","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:40.533Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"169f1613-4107-4b95-9cd8-86dd808d12d7"}
{"level":"error","ts":"2026-07-13T15:51:40.533Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"169f1613-4107-4b95-9cd8-86dd808d12d7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:40.534Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"ab03cd7e-5ffe-4c89-b5b5-8cf421b86842","rayCluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.537Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"45ea3c61-1e18-4414-907a-617a5bd9343f","name":"kube-rbac-proxy-config-stop-bvlzx","operation":"created"}
{"level":"info","ts":"2026-07-13T15:51:40.538Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"7acf378e-52e0-440d-9fee-f964a4e2baea","name":"stop-bvlzx-workers"}
{"level":"info","ts":"2026-07-13T15:51:40.538Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"7acf378e-52e0-440d-9fee-f964a4e2baea","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.538Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"c6919edd-17a4-4f66-ab1e-577404beb0bb","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.622Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"ab03cd7e-5ffe-4c89-b5b5-8cf421b86842","rayCluster":"stop-bvlzx","podIPs":[]}
{"level":"error","ts":"2026-07-13T15:51:40.622Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"45ea3c61-1e18-4414-907a-617a5bd9343f","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-bvlzx\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:40.622Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"45ea3c61-1e18-4414-907a-617a5bd9343f"}
{"level":"error","ts":"2026-07-13T15:51:40.622Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"45ea3c61-1e18-4414-907a-617a5bd9343f","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-bvlzx\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:40.622Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1b247715-621a-4bc6-9f1a-03585fe46312"}
{"level":"info","ts":"2026-07-13T15:51:40.622Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"9c29fb09-aa4f-4897-855a-6218fac59c34","namespacedName":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:40.622Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"9c29fb09-aa4f-4897-855a-6218fac59c34","mode":"IntegratedOAuth","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.622Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"9c29fb09-aa4f-4897-855a-6218fac59c34","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.622Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1b247715-621a-4bc6-9f1a-03585fe46312"}
{"level":"info","ts":"2026-07-13T15:51:40.623Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1b247715-621a-4bc6-9f1a-03585fe46312","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:40.623Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1b247715-621a-4bc6-9f1a-03585fe46312","error":"head certificate secret ray-head-secret-stop-bvlzx not found"}
{"level":"error","ts":"2026-07-13T15:51:40.623Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1b247715-621a-4bc6-9f1a-03585fe46312","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:40.623Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1b247715-621a-4bc6-9f1a-03585fe46312","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found"}
{"level":"info","ts":"2026-07-13T15:51:40.623Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1b247715-621a-4bc6-9f1a-03585fe46312","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:40.632Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"c6919edd-17a4-4f66-ab1e-577404beb0bb","name":"stop-bvlzx-head"}
{"level":"info","ts":"2026-07-13T15:51:40.623Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1b247715-621a-4bc6-9f1a-03585fe46312","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:40.721Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1b247715-621a-4bc6-9f1a-03585fe46312","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:40.721Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1b247715-621a-4bc6-9f1a-03585fe46312"}
{"level":"error","ts":"2026-07-13T15:51:40.721Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1b247715-621a-4bc6-9f1a-03585fe46312","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:40.722Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"9c29fb09-aa4f-4897-855a-6218fac59c34","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.722Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"2ad6ed9a-a099-4115-b0ca-7c23db5ba1fb","namespacedName":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:40.722Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"2ad6ed9a-a099-4115-b0ca-7c23db5ba1fb","mode":"IntegratedOAuth","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.722Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"2ad6ed9a-a099-4115-b0ca-7c23db5ba1fb","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.821Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3a0b83e5-9087-4c71-bef6-3e1abbcfe37e"}
{"level":"info","ts":"2026-07-13T15:51:40.821Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3a0b83e5-9087-4c71-bef6-3e1abbcfe37e"}
{"level":"info","ts":"2026-07-13T15:51:40.821Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3a0b83e5-9087-4c71-bef6-3e1abbcfe37e","1 head service found":"stop-bvlzx-head-svc"}
{"level":"error","ts":"2026-07-13T15:51:40.821Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"2ad6ed9a-a099-4115-b0ca-7c23db5ba1fb","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-bvlzx\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:40.821Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3a0b83e5-9087-4c71-bef6-3e1abbcfe37e","error":"head certificate secret ray-head-secret-stop-bvlzx not found"}
{"level":"info","ts":"2026-07-13T15:51:40.821Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"2ad6ed9a-a099-4115-b0ca-7c23db5ba1fb"}
{"level":"error","ts":"2026-07-13T15:51:40.821Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"2ad6ed9a-a099-4115-b0ca-7c23db5ba1fb","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-bvlzx\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:51:40.821Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3a0b83e5-9087-4c71-bef6-3e1abbcfe37e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:40.821Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3a0b83e5-9087-4c71-bef6-3e1abbcfe37e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found"}
{"level":"info","ts":"2026-07-13T15:51:40.821Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"c6919edd-17a4-4f66-ab1e-577404beb0bb","name":"stop-bvlzx-workers"}
{"level":"info","ts":"2026-07-13T15:51:40.822Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"c6919edd-17a4-4f66-ab1e-577404beb0bb","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.821Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3a0b83e5-9087-4c71-bef6-3e1abbcfe37e","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:40.822Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3a0b83e5-9087-4c71-bef6-3e1abbcfe37e","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:40.822Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3a0b83e5-9087-4c71-bef6-3e1abbcfe37e","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:40.822Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"26521d56-7a50-41aa-b834-42cf98db4d16","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.822Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"1c3c57a7-99f7-4213-89c8-6e7831999d54","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:40.822Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"1c3c57a7-99f7-4213-89c8-6e7831999d54","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.822Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"1c3c57a7-99f7-4213-89c8-6e7831999d54","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:40.822Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3a0b83e5-9087-4c71-bef6-3e1abbcfe37e"}
{"level":"error","ts":"2026-07-13T15:51:40.822Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3a0b83e5-9087-4c71-bef6-3e1abbcfe37e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:40.822Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"1c3c57a7-99f7-4213-89c8-6e7831999d54"}
{"level":"info","ts":"2026-07-13T15:51:40.822Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"1c3c57a7-99f7-4213-89c8-6e7831999d54","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:51:40.921Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"4436235b-c7c0-4374-b903-e9c1260d1929","namespacedName":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:40.921Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"4436235b-c7c0-4374-b903-e9c1260d1929","mode":"IntegratedOAuth","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.921Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"4436235b-c7c0-4374-b903-e9c1260d1929","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.922Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"ab03cd7e-5ffe-4c89-b5b5-8cf421b86842","rayCluster":"stop-bvlzx","podIPs":[]}
{"level":"error","ts":"2026-07-13T15:51:40.922Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"ab03cd7e-5ffe-4c89-b5b5-8cf421b86842","error":"Certificate.cert-manager.io \"ray-worker-cert-stop-bvlzx\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:40.922Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"ab03cd7e-5ffe-4c89-b5b5-8cf421b86842"}
{"level":"error","ts":"2026-07-13T15:51:40.922Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"ab03cd7e-5ffe-4c89-b5b5-8cf421b86842","error":"Certificate.cert-manager.io \"ray-worker-cert-stop-bvlzx\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:40.922Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"4fd19934-164b-4c57-8f0b-04c5196f9acb","rayCluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.922Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"4fd19934-164b-4c57-8f0b-04c5196f9acb"}
{"level":"info","ts":"2026-07-13T15:51:40.925Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"26521d56-7a50-41aa-b834-42cf98db4d16","name":"stop-bvlzx-head"}
{"level":"info","ts":"2026-07-13T15:51:40.927Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"c8f07651-6a37-4ae2-b969-668891535fd2","rayCluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:40.927Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"c8f07651-6a37-4ae2-b969-668891535fd2"}
{"level":"info","ts":"2026-07-13T15:51:40.934Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"4436235b-c7c0-4374-b903-e9c1260d1929","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:41.021Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"34b02319-657f-40a8-91b1-aa932a8c0d71"}
{"level":"info","ts":"2026-07-13T15:51:41.021Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"34b02319-657f-40a8-91b1-aa932a8c0d71"}
{"level":"info","ts":"2026-07-13T15:51:41.021Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"34b02319-657f-40a8-91b1-aa932a8c0d71","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:41.021Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"34b02319-657f-40a8-91b1-aa932a8c0d71","error":"head certificate secret ray-head-secret-stop-bvlzx not found"}
{"level":"error","ts":"2026-07-13T15:51:41.021Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"34b02319-657f-40a8-91b1-aa932a8c0d71","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:41.021Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"34b02319-657f-40a8-91b1-aa932a8c0d71","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found"}
{"level":"info","ts":"2026-07-13T15:51:41.021Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"34b02319-657f-40a8-91b1-aa932a8c0d71","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:41.021Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"34b02319-657f-40a8-91b1-aa932a8c0d71","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:41.021Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"34b02319-657f-40a8-91b1-aa932a8c0d71","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:41.021Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"34b02319-657f-40a8-91b1-aa932a8c0d71"}
{"level":"error","ts":"2026-07-13T15:51:41.021Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"34b02319-657f-40a8-91b1-aa932a8c0d71","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:41.025Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"26521d56-7a50-41aa-b834-42cf98db4d16","name":"stop-bvlzx-workers"}
{"level":"info","ts":"2026-07-13T15:51:41.025Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"26521d56-7a50-41aa-b834-42cf98db4d16","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:41.342Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"f33193f6-5348-468f-b8a2-6f1c68515060"}
{"level":"info","ts":"2026-07-13T15:51:41.342Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"f33193f6-5348-468f-b8a2-6f1c68515060"}
{"level":"info","ts":"2026-07-13T15:51:41.342Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"f33193f6-5348-468f-b8a2-6f1c68515060","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:41.342Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"f33193f6-5348-468f-b8a2-6f1c68515060","error":"head certificate secret ray-head-secret-stop-bvlzx not found"}
{"level":"error","ts":"2026-07-13T15:51:41.342Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"f33193f6-5348-468f-b8a2-6f1c68515060","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:41.342Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"f33193f6-5348-468f-b8a2-6f1c68515060","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found"}
{"level":"info","ts":"2026-07-13T15:51:41.342Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"f33193f6-5348-468f-b8a2-6f1c68515060","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:41.342Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"f33193f6-5348-468f-b8a2-6f1c68515060","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:41.342Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"f33193f6-5348-468f-b8a2-6f1c68515060","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:41.342Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"f33193f6-5348-468f-b8a2-6f1c68515060"}
{"level":"error","ts":"2026-07-13T15:51:41.342Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"f33193f6-5348-468f-b8a2-6f1c68515060","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:41.983Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ca247693-9ef4-4786-8295-25946c38253d"}
{"level":"info","ts":"2026-07-13T15:51:41.983Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ca247693-9ef4-4786-8295-25946c38253d"}
{"level":"info","ts":"2026-07-13T15:51:41.983Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ca247693-9ef4-4786-8295-25946c38253d","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:41.983Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ca247693-9ef4-4786-8295-25946c38253d","error":"head certificate secret ray-head-secret-stop-bvlzx not found"}
{"level":"error","ts":"2026-07-13T15:51:41.983Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ca247693-9ef4-4786-8295-25946c38253d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:41.983Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ca247693-9ef4-4786-8295-25946c38253d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found"}
{"level":"info","ts":"2026-07-13T15:51:41.983Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ca247693-9ef4-4786-8295-25946c38253d","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:41.983Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ca247693-9ef4-4786-8295-25946c38253d","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:41.983Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ca247693-9ef4-4786-8295-25946c38253d","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"stop-bvlzx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:51:41.983Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ca247693-9ef4-4786-8295-25946c38253d"}
{"level":"error","ts":"2026-07-13T15:51:41.983Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"ca247693-9ef4-4786-8295-25946c38253d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-bvlzx not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:43.264Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd"}
{"level":"info","ts":"2026-07-13T15:51:43.264Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd"}
{"level":"info","ts":"2026-07-13T15:51:43.264Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:43.264Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd"}
{"level":"info","ts":"2026-07-13T15:51:43.264Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:51:43.264Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd"}
{"level":"info","ts":"2026-07-13T15:51:43.264Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","mode":"IntegratedOAuth","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:43.264Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","cluster":"stop-bvlzx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:51:43.264Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","cluster":"stop-bvlzx","authType":"IntegratedOAuth","serviceAccount":"stop-bvlzx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T15:51:43.264Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"stop-bvlzx","ray.io/group":"headgroup","ray.io/identifier":"stop-bvlzx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T15:51:43.264Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T15:51:43.264Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:51:43.264Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:51:43.264Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","enabled":true}
{"level":"info","ts":"2026-07-13T15:51:43.280Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"c4e29bdd-c1cb-410f-a200-e55879b1760f","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:43.280Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"c4e29bdd-c1cb-410f-a200-e55879b1760f","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 15:51:40 +0000 UTC","Now":"2026-07-13T15:51:43.280Z","ShutdownTime":"2026-07-13T15:51:40.000Z"}
{"level":"info","ts":"2026-07-13T15:51:43.283Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","name":"stop-bvlzx-head-cwv4c"}
{"level":"info","ts":"2026-07-13T15:51:43.283Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:51:43.283Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:43.283Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T15:51:43.283Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:43.283Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T15:51:43.283Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd"}
{"level":"info","ts":"2026-07-13T15:51:43.283Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","nodeType":"worker","rayStartParams":{"address":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T15:51:43.283Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","rayStartCmd":"ray start  --address=stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:51:43.284Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:51:43.284Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","enabled":true}
{"level":"info","ts":"2026-07-13T15:51:43.297Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","name":"stop-bvlzx-small-group-worker-r2kxc"}
{"level":"info","ts":"2026-07-13T15:51:43.297Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","podName":"stop-bvlzx-head-cwv4c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:43.297Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","podName":"stop-bvlzx-head-cwv4c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:43.297Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"stop-bvlzx-head-svc"},"newHead":{"podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"}}
{"level":"info","ts":"2026-07-13T15:51:43.297Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23750da1-ac3c-4f39-8d29-d7dba82754cd","name":"stop-bvlzx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:51:43.309Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23cd30aa-c4fe-42c0-85f5-78a2ec42ffbe"}
{"level":"info","ts":"2026-07-13T15:51:43.309Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23cd30aa-c4fe-42c0-85f5-78a2ec42ffbe"}
{"level":"info","ts":"2026-07-13T15:51:43.309Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23cd30aa-c4fe-42c0-85f5-78a2ec42ffbe","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:43.309Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"6110fe1a-8975-483b-b47d-b75e6d448f25","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:43.309Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"0c2fb873-3c81-4b04-bd8d-8d73ec18e292","rayCluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:43.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23cd30aa-c4fe-42c0-85f5-78a2ec42ffbe","Found 1 head Pod":"stop-bvlzx-head-cwv4c","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:51:43.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23cd30aa-c4fe-42c0-85f5-78a2ec42ffbe","head Pod":"stop-bvlzx-head-cwv4c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-bvlzx-head-cwv4c. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:43.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23cd30aa-c4fe-42c0-85f5-78a2ec42ffbe","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:51:43.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23cd30aa-c4fe-42c0-85f5-78a2ec42ffbe","worker Pod":"stop-bvlzx-small-group-worker-r2kxc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-bvlzx-small-group-worker-r2kxc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:43.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23cd30aa-c4fe-42c0-85f5-78a2ec42ffbe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:43.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23cd30aa-c4fe-42c0-85f5-78a2ec42ffbe","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:51:43.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23cd30aa-c4fe-42c0-85f5-78a2ec42ffbe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:43.309Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"0c2fb873-3c81-4b04-bd8d-8d73ec18e292","rayCluster":"stop-bvlzx","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:51:43.309Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23cd30aa-c4fe-42c0-85f5-78a2ec42ffbe","podName":"stop-bvlzx-head-cwv4c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:43.309Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23cd30aa-c4fe-42c0-85f5-78a2ec42ffbe","podName":"stop-bvlzx-head-cwv4c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:43.309Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23cd30aa-c4fe-42c0-85f5-78a2ec42ffbe","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"stop-bvlzx-head-svc"},"newHead":{"podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"}}
{"level":"info","ts":"2026-07-13T15:51:43.309Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23cd30aa-c4fe-42c0-85f5-78a2ec42ffbe","name":"stop-bvlzx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:51:43.309Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b834f24a-dbd0-4375-9b2e-7a6fdffe196a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:43.309Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b834f24a-dbd0-4375-9b2e-7a6fdffe196a","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:43.310Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b834f24a-dbd0-4375-9b2e-7a6fdffe196a","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:43.310Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b834f24a-dbd0-4375-9b2e-7a6fdffe196a"}
{"level":"info","ts":"2026-07-13T15:51:43.310Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b834f24a-dbd0-4375-9b2e-7a6fdffe196a","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:51:43.328Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23cd30aa-c4fe-42c0-85f5-78a2ec42ffbe","name":"stop-bvlzx","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-bvlzx\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-5xvz7","name":"stop-bvlzx"}}
{"level":"info","ts":"2026-07-13T15:51:43.328Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23cd30aa-c4fe-42c0-85f5-78a2ec42ffbe"}
{"level":"error","ts":"2026-07-13T15:51:43.328Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"23cd30aa-c4fe-42c0-85f5-78a2ec42ffbe","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-bvlzx\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:43.328Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2c0eff54-a531-4761-878c-65a797fc1f4e"}
{"level":"info","ts":"2026-07-13T15:51:43.328Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2c0eff54-a531-4761-878c-65a797fc1f4e"}
{"level":"info","ts":"2026-07-13T15:51:43.328Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2c0eff54-a531-4761-878c-65a797fc1f4e","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:43.328Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2c0eff54-a531-4761-878c-65a797fc1f4e","Found 1 head Pod":"stop-bvlzx-head-cwv4c","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:51:43.328Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2c0eff54-a531-4761-878c-65a797fc1f4e","head Pod":"stop-bvlzx-head-cwv4c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-bvlzx-head-cwv4c. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:43.328Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2c0eff54-a531-4761-878c-65a797fc1f4e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:51:43.328Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2c0eff54-a531-4761-878c-65a797fc1f4e","worker Pod":"stop-bvlzx-small-group-worker-r2kxc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-bvlzx-small-group-worker-r2kxc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:43.328Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2c0eff54-a531-4761-878c-65a797fc1f4e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:43.328Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2c0eff54-a531-4761-878c-65a797fc1f4e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:51:43.328Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2c0eff54-a531-4761-878c-65a797fc1f4e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:43.328Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2c0eff54-a531-4761-878c-65a797fc1f4e","podName":"stop-bvlzx-head-cwv4c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:43.328Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2c0eff54-a531-4761-878c-65a797fc1f4e","podName":"stop-bvlzx-head-cwv4c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:43.328Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2c0eff54-a531-4761-878c-65a797fc1f4e","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:51:43.328Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2c0eff54-a531-4761-878c-65a797fc1f4e","name":"stop-bvlzx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:51:43.334Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"6110fe1a-8975-483b-b47d-b75e6d448f25","name":"stop-bvlzx-head"}
{"level":"info","ts":"2026-07-13T15:51:43.339Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8fa6bab2-87a9-456d-a243-8f75641c5ad2"}
{"level":"info","ts":"2026-07-13T15:51:43.339Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8fa6bab2-87a9-456d-a243-8f75641c5ad2"}
{"level":"info","ts":"2026-07-13T15:51:43.339Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8fa6bab2-87a9-456d-a243-8f75641c5ad2","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:43.339Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8fa6bab2-87a9-456d-a243-8f75641c5ad2","Found 1 head Pod":"stop-bvlzx-head-cwv4c","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:51:43.339Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8fa6bab2-87a9-456d-a243-8f75641c5ad2","head Pod":"stop-bvlzx-head-cwv4c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-bvlzx-head-cwv4c. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:43.340Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8fa6bab2-87a9-456d-a243-8f75641c5ad2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:51:43.340Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8fa6bab2-87a9-456d-a243-8f75641c5ad2","worker Pod":"stop-bvlzx-small-group-worker-r2kxc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-bvlzx-small-group-worker-r2kxc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:43.340Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8fa6bab2-87a9-456d-a243-8f75641c5ad2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:43.340Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8fa6bab2-87a9-456d-a243-8f75641c5ad2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:51:43.340Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8fa6bab2-87a9-456d-a243-8f75641c5ad2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:43.340Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8fa6bab2-87a9-456d-a243-8f75641c5ad2","podName":"stop-bvlzx-head-cwv4c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:43.340Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8fa6bab2-87a9-456d-a243-8f75641c5ad2","podName":"stop-bvlzx-head-cwv4c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:43.340Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8fa6bab2-87a9-456d-a243-8f75641c5ad2","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:51:43.340Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8fa6bab2-87a9-456d-a243-8f75641c5ad2","name":"stop-bvlzx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:51:43.340Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"209f9f41-b872-441c-8c31-4c55dc560725","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:43.340Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"209f9f41-b872-441c-8c31-4c55dc560725","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:43.340Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"209f9f41-b872-441c-8c31-4c55dc560725","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:43.340Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"40904e44-8e2f-4b39-9bc2-563d146c3011","rayCluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:43.340Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"209f9f41-b872-441c-8c31-4c55dc560725"}
{"level":"info","ts":"2026-07-13T15:51:43.340Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"209f9f41-b872-441c-8c31-4c55dc560725","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:51:43.340Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"40904e44-8e2f-4b39-9bc2-563d146c3011","rayCluster":"stop-bvlzx","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:51:43.345Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"6110fe1a-8975-483b-b47d-b75e6d448f25","name":"stop-bvlzx-workers"}
{"level":"info","ts":"2026-07-13T15:51:43.345Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"6110fe1a-8975-483b-b47d-b75e6d448f25","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:43.345Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"0936d9e5-ee33-483f-9c19-95ec09d3c09b","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:43.345Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8fa6bab2-87a9-456d-a243-8f75641c5ad2","name":"stop-bvlzx","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-bvlzx\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-5xvz7","name":"stop-bvlzx"}}
{"level":"info","ts":"2026-07-13T15:51:43.345Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8fa6bab2-87a9-456d-a243-8f75641c5ad2"}
{"level":"error","ts":"2026-07-13T15:51:43.345Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8fa6bab2-87a9-456d-a243-8f75641c5ad2","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-bvlzx\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:51:43.350Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"e6a2898d-a3ce-49b5-895d-05e1844e8f5c"}
{"level":"info","ts":"2026-07-13T15:51:43.350Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"e6a2898d-a3ce-49b5-895d-05e1844e8f5c"}
{"level":"info","ts":"2026-07-13T15:51:43.350Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"e6a2898d-a3ce-49b5-895d-05e1844e8f5c","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:43.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"e6a2898d-a3ce-49b5-895d-05e1844e8f5c","Found 1 head Pod":"stop-bvlzx-head-cwv4c","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:51:43.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"e6a2898d-a3ce-49b5-895d-05e1844e8f5c","head Pod":"stop-bvlzx-head-cwv4c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-bvlzx-head-cwv4c. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:43.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"e6a2898d-a3ce-49b5-895d-05e1844e8f5c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:51:43.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"e6a2898d-a3ce-49b5-895d-05e1844e8f5c","worker Pod":"stop-bvlzx-small-group-worker-r2kxc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-bvlzx-small-group-worker-r2kxc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:43.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"e6a2898d-a3ce-49b5-895d-05e1844e8f5c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:43.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"e6a2898d-a3ce-49b5-895d-05e1844e8f5c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:51:43.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"e6a2898d-a3ce-49b5-895d-05e1844e8f5c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:43.351Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"e6a2898d-a3ce-49b5-895d-05e1844e8f5c","podName":"stop-bvlzx-small-group-worker-r2kxc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:43.351Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"e6a2898d-a3ce-49b5-895d-05e1844e8f5c","podName":"stop-bvlzx-small-group-worker-r2kxc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:43.351Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"e6a2898d-a3ce-49b5-895d-05e1844e8f5c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:51:43.351Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"e6a2898d-a3ce-49b5-895d-05e1844e8f5c","seconds":300}
{"level":"info","ts":"2026-07-13T15:51:43.354Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"0936d9e5-ee33-483f-9c19-95ec09d3c09b","name":"stop-bvlzx-head"}
{"level":"info","ts":"2026-07-13T15:51:43.365Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"0936d9e5-ee33-483f-9c19-95ec09d3c09b","name":"stop-bvlzx-workers"}
{"level":"info","ts":"2026-07-13T15:51:43.365Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"0936d9e5-ee33-483f-9c19-95ec09d3c09b","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:43.478Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"5b292422-1b84-4ede-8885-fae4cf43ff7f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:43.478Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"5b292422-1b84-4ede-8885-fae4cf43ff7f","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:43.478Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"5b292422-1b84-4ede-8885-fae4cf43ff7f","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:43.478Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"5b292422-1b84-4ede-8885-fae4cf43ff7f"}
{"level":"info","ts":"2026-07-13T15:51:43.478Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"5b292422-1b84-4ede-8885-fae4cf43ff7f","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:51:43.735Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"0351110e-01f1-48f7-996d-0b9c917290c4"}
{"level":"info","ts":"2026-07-13T15:51:43.735Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"0351110e-01f1-48f7-996d-0b9c917290c4"}
{"level":"info","ts":"2026-07-13T15:51:43.735Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"0351110e-01f1-48f7-996d-0b9c917290c4","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:43.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"0351110e-01f1-48f7-996d-0b9c917290c4","Found 1 head Pod":"stop-bvlzx-head-cwv4c","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:51:43.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"0351110e-01f1-48f7-996d-0b9c917290c4","head Pod":"stop-bvlzx-head-cwv4c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-bvlzx-head-cwv4c. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:43.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"0351110e-01f1-48f7-996d-0b9c917290c4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:51:43.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"0351110e-01f1-48f7-996d-0b9c917290c4","worker Pod":"stop-bvlzx-small-group-worker-r2kxc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-bvlzx-small-group-worker-r2kxc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:43.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"0351110e-01f1-48f7-996d-0b9c917290c4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:43.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"0351110e-01f1-48f7-996d-0b9c917290c4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:51:43.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"0351110e-01f1-48f7-996d-0b9c917290c4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:43.735Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"0351110e-01f1-48f7-996d-0b9c917290c4","podName":"stop-bvlzx-head-cwv4c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:43.735Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"0351110e-01f1-48f7-996d-0b9c917290c4","podName":"stop-bvlzx-head-cwv4c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:43.735Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"0351110e-01f1-48f7-996d-0b9c917290c4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:51:43.735Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"0351110e-01f1-48f7-996d-0b9c917290c4","seconds":300}
{"level":"info","ts":"2026-07-13T15:51:43.751Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"11e0207c-ac2f-4e95-9319-01a29074c7d4"}
{"level":"info","ts":"2026-07-13T15:51:43.751Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"11e0207c-ac2f-4e95-9319-01a29074c7d4"}
{"level":"info","ts":"2026-07-13T15:51:43.751Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"11e0207c-ac2f-4e95-9319-01a29074c7d4","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:43.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"11e0207c-ac2f-4e95-9319-01a29074c7d4","Found 1 head Pod":"stop-bvlzx-head-cwv4c","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:51:43.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"11e0207c-ac2f-4e95-9319-01a29074c7d4","head Pod":"stop-bvlzx-head-cwv4c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-bvlzx-head-cwv4c. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:43.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"11e0207c-ac2f-4e95-9319-01a29074c7d4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:51:43.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"11e0207c-ac2f-4e95-9319-01a29074c7d4","worker Pod":"stop-bvlzx-small-group-worker-r2kxc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-bvlzx-small-group-worker-r2kxc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:43.752Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"11e0207c-ac2f-4e95-9319-01a29074c7d4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:43.752Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"11e0207c-ac2f-4e95-9319-01a29074c7d4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:51:43.752Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"11e0207c-ac2f-4e95-9319-01a29074c7d4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:43.752Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"11e0207c-ac2f-4e95-9319-01a29074c7d4","podName":"stop-bvlzx-head-cwv4c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:43.752Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"11e0207c-ac2f-4e95-9319-01a29074c7d4","podName":"stop-bvlzx-head-cwv4c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:43.752Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"11e0207c-ac2f-4e95-9319-01a29074c7d4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:51:43.752Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"11e0207c-ac2f-4e95-9319-01a29074c7d4","seconds":300}
{"level":"info","ts":"2026-07-13T15:51:44.288Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1df652d9-c45e-44cb-9b37-a5a7b5f914e7"}
{"level":"info","ts":"2026-07-13T15:51:44.288Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1df652d9-c45e-44cb-9b37-a5a7b5f914e7"}
{"level":"info","ts":"2026-07-13T15:51:44.288Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1df652d9-c45e-44cb-9b37-a5a7b5f914e7","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:44.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1df652d9-c45e-44cb-9b37-a5a7b5f914e7","Found 1 head Pod":"stop-bvlzx-head-cwv4c","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:51:44.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1df652d9-c45e-44cb-9b37-a5a7b5f914e7","head Pod":"stop-bvlzx-head-cwv4c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-bvlzx-head-cwv4c. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:44.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1df652d9-c45e-44cb-9b37-a5a7b5f914e7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:51:44.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1df652d9-c45e-44cb-9b37-a5a7b5f914e7","worker Pod":"stop-bvlzx-small-group-worker-r2kxc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-bvlzx-small-group-worker-r2kxc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:44.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1df652d9-c45e-44cb-9b37-a5a7b5f914e7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:44.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1df652d9-c45e-44cb-9b37-a5a7b5f914e7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:51:44.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1df652d9-c45e-44cb-9b37-a5a7b5f914e7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:44.289Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1df652d9-c45e-44cb-9b37-a5a7b5f914e7","podName":"stop-bvlzx-head-cwv4c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:44.289Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1df652d9-c45e-44cb-9b37-a5a7b5f914e7","podName":"stop-bvlzx-head-cwv4c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:44.289Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1df652d9-c45e-44cb-9b37-a5a7b5f914e7","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"newHead":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"}}
{"level":"info","ts":"2026-07-13T15:51:44.289Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1df652d9-c45e-44cb-9b37-a5a7b5f914e7","name":"stop-bvlzx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:51:44.300Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"b6fc3f5b-9559-4e4d-848f-ad2ca9befda5","rayCluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:44.300Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"69ded6ac-d763-4eca-b233-a980bf65f19b","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:44.300Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"b6fc3f5b-9559-4e4d-848f-ad2ca9befda5","pod":"stop-bvlzx-head-cwv4c","ip":"10.133.0.21"}
{"level":"info","ts":"2026-07-13T15:51:44.300Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"ecffac84-5ace-4434-99fc-bc7b732e5770","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:44.300Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"ecffac84-5ace-4434-99fc-bc7b732e5770","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:44.300Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"ecffac84-5ace-4434-99fc-bc7b732e5770","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:44.300Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"ecffac84-5ace-4434-99fc-bc7b732e5770"}
{"level":"info","ts":"2026-07-13T15:51:44.300Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"ecffac84-5ace-4434-99fc-bc7b732e5770","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:51:44.311Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"69ded6ac-d763-4eca-b233-a980bf65f19b","name":"stop-bvlzx-head"}
{"level":"info","ts":"2026-07-13T15:51:44.311Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"b6fc3f5b-9559-4e4d-848f-ad2ca9befda5","certificate":"ray-head-cert-stop-bvlzx","dnsNames":["stop-bvlzx-head-svc","localhost","stop-bvlzx-head-svc.test-ns-5xvz7.svc","stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local"],"ipAddresses":["10.133.0.21","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:51:44.322Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"69ded6ac-d763-4eca-b233-a980bf65f19b","name":"stop-bvlzx-workers"}
{"level":"info","ts":"2026-07-13T15:51:44.322Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"69ded6ac-d763-4eca-b233-a980bf65f19b","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:44.322Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"b6fc3f5b-9559-4e4d-848f-ad2ca9befda5","certificate":"ray-worker-cert-stop-bvlzx","dnsNames":["stop-bvlzx-worker-svc","localhost","stop-bvlzx-small-group","stop-bvlzx-worker-svc.test-ns-5xvz7.svc","stop-bvlzx-worker-svc.test-ns-5xvz7.svc.cluster.local","*.stop-bvlzx-worker-svc.test-ns-5xvz7.svc","*.stop-bvlzx-worker-svc.test-ns-5xvz7.svc.cluster.local","*-worker-*.test-ns-5xvz7.svc","*-worker-*.test-ns-5xvz7.svc.cluster.local"],"ipAddresses":["10.133.0.21","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:51:44.322Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"b6fc3f5b-9559-4e4d-848f-ad2ca9befda5","rayCluster":"stop-bvlzx","podIPs":["10.133.0.21"]}
{"level":"info","ts":"2026-07-13T15:51:44.400Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06d6f788-5db1-41ec-9c50-97a7fa9a49f2"}
{"level":"info","ts":"2026-07-13T15:51:44.400Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06d6f788-5db1-41ec-9c50-97a7fa9a49f2"}
{"level":"info","ts":"2026-07-13T15:51:44.400Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06d6f788-5db1-41ec-9c50-97a7fa9a49f2","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:44.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06d6f788-5db1-41ec-9c50-97a7fa9a49f2","Found 1 head Pod":"stop-bvlzx-head-cwv4c","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:51:44.401Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06d6f788-5db1-41ec-9c50-97a7fa9a49f2","head Pod":"stop-bvlzx-head-cwv4c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-bvlzx-head-cwv4c. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:44.401Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06d6f788-5db1-41ec-9c50-97a7fa9a49f2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:51:44.401Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06d6f788-5db1-41ec-9c50-97a7fa9a49f2","worker Pod":"stop-bvlzx-small-group-worker-r2kxc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-bvlzx-small-group-worker-r2kxc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:44.401Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06d6f788-5db1-41ec-9c50-97a7fa9a49f2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:44.401Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06d6f788-5db1-41ec-9c50-97a7fa9a49f2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:51:44.401Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06d6f788-5db1-41ec-9c50-97a7fa9a49f2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:44.401Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06d6f788-5db1-41ec-9c50-97a7fa9a49f2","podName":"stop-bvlzx-head-cwv4c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:44.401Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06d6f788-5db1-41ec-9c50-97a7fa9a49f2","podName":"stop-bvlzx-head-cwv4c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:44.401Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06d6f788-5db1-41ec-9c50-97a7fa9a49f2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:51:44.401Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06d6f788-5db1-41ec-9c50-97a7fa9a49f2","seconds":300}
{"level":"info","ts":"2026-07-13T15:51:46.300Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1934fc6d-ccbb-49fd-a6df-80e95547afac"}
{"level":"info","ts":"2026-07-13T15:51:46.300Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1934fc6d-ccbb-49fd-a6df-80e95547afac"}
{"level":"info","ts":"2026-07-13T15:51:46.300Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1934fc6d-ccbb-49fd-a6df-80e95547afac","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:46.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1934fc6d-ccbb-49fd-a6df-80e95547afac","Found 1 head Pod":"stop-bvlzx-head-cwv4c","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:51:46.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1934fc6d-ccbb-49fd-a6df-80e95547afac","head Pod":"stop-bvlzx-head-cwv4c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-bvlzx-head-cwv4c. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:46.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1934fc6d-ccbb-49fd-a6df-80e95547afac","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:51:46.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1934fc6d-ccbb-49fd-a6df-80e95547afac","worker Pod":"stop-bvlzx-small-group-worker-r2kxc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-bvlzx-small-group-worker-r2kxc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:46.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1934fc6d-ccbb-49fd-a6df-80e95547afac","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:46.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1934fc6d-ccbb-49fd-a6df-80e95547afac","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:51:46.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1934fc6d-ccbb-49fd-a6df-80e95547afac","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:46.300Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1934fc6d-ccbb-49fd-a6df-80e95547afac","podName":"stop-bvlzx-head-cwv4c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:46.300Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1934fc6d-ccbb-49fd-a6df-80e95547afac","podName":"stop-bvlzx-head-cwv4c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:46.300Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1934fc6d-ccbb-49fd-a6df-80e95547afac","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:51:46.300Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"1934fc6d-ccbb-49fd-a6df-80e95547afac","seconds":300}
{"level":"info","ts":"2026-07-13T15:51:46.478Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"796176c3-20f5-4dba-8bc6-1528b6b66062","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:46.479Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"796176c3-20f5-4dba-8bc6-1528b6b66062","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:46.479Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"796176c3-20f5-4dba-8bc6-1528b6b66062","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:46.479Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"796176c3-20f5-4dba-8bc6-1528b6b66062"}
{"level":"info","ts":"2026-07-13T15:51:46.479Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"796176c3-20f5-4dba-8bc6-1528b6b66062","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:51:49.332Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8c3381d3-9fcf-431b-a5de-9865c37e9b63"}
{"level":"info","ts":"2026-07-13T15:51:49.332Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8c3381d3-9fcf-431b-a5de-9865c37e9b63"}
{"level":"info","ts":"2026-07-13T15:51:49.332Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8c3381d3-9fcf-431b-a5de-9865c37e9b63","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:49.332Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8c3381d3-9fcf-431b-a5de-9865c37e9b63","Found 1 head Pod":"stop-bvlzx-head-cwv4c","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:51:49.332Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8c3381d3-9fcf-431b-a5de-9865c37e9b63","head Pod":"stop-bvlzx-head-cwv4c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-bvlzx-head-cwv4c. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:49.332Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8c3381d3-9fcf-431b-a5de-9865c37e9b63","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:51:49.332Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8c3381d3-9fcf-431b-a5de-9865c37e9b63","worker Pod":"stop-bvlzx-small-group-worker-r2kxc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-bvlzx-small-group-worker-r2kxc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:49.332Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8c3381d3-9fcf-431b-a5de-9865c37e9b63","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:49.332Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8c3381d3-9fcf-431b-a5de-9865c37e9b63","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:51:49.332Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8c3381d3-9fcf-431b-a5de-9865c37e9b63","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:49.332Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8c3381d3-9fcf-431b-a5de-9865c37e9b63","podName":"stop-bvlzx-head-cwv4c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:49.332Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8c3381d3-9fcf-431b-a5de-9865c37e9b63","podName":"stop-bvlzx-head-cwv4c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:51:49.332Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8c3381d3-9fcf-431b-a5de-9865c37e9b63","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:51:49.332Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"8c3381d3-9fcf-431b-a5de-9865c37e9b63","seconds":300}
{"level":"info","ts":"2026-07-13T15:51:49.480Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"3b9ccb63-c6f4-4b01-b80c-159f19f5f6de","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:49.480Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"3b9ccb63-c6f4-4b01-b80c-159f19f5f6de","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:49.480Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"3b9ccb63-c6f4-4b01-b80c-159f19f5f6de","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:49.480Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"3b9ccb63-c6f4-4b01-b80c-159f19f5f6de"}
{"level":"info","ts":"2026-07-13T15:51:49.480Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"3b9ccb63-c6f4-4b01-b80c-159f19f5f6de","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:51:50.318Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5bf62499-abda-44e5-a432-23a9f7b26ddf"}
{"level":"info","ts":"2026-07-13T15:51:50.318Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5bf62499-abda-44e5-a432-23a9f7b26ddf"}
{"level":"info","ts":"2026-07-13T15:51:50.318Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5bf62499-abda-44e5-a432-23a9f7b26ddf","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:50.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5bf62499-abda-44e5-a432-23a9f7b26ddf","Found 1 head Pod":"stop-bvlzx-head-cwv4c","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:51:50.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5bf62499-abda-44e5-a432-23a9f7b26ddf","head Pod":"stop-bvlzx-head-cwv4c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-bvlzx-head-cwv4c. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:50.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5bf62499-abda-44e5-a432-23a9f7b26ddf","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:51:50.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5bf62499-abda-44e5-a432-23a9f7b26ddf","worker Pod":"stop-bvlzx-small-group-worker-r2kxc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-bvlzx-small-group-worker-r2kxc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:50.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5bf62499-abda-44e5-a432-23a9f7b26ddf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:50.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5bf62499-abda-44e5-a432-23a9f7b26ddf","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:51:50.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5bf62499-abda-44e5-a432-23a9f7b26ddf","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:50.318Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5bf62499-abda-44e5-a432-23a9f7b26ddf","podName":"stop-bvlzx-head-cwv4c","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:51:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:51:50.318Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5bf62499-abda-44e5-a432-23a9f7b26ddf","podName":"stop-bvlzx-head-cwv4c","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:51:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:51:50.318Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5bf62499-abda-44e5-a432-23a9f7b26ddf","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:51:50.318Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5bf62499-abda-44e5-a432-23a9f7b26ddf","name":"stop-bvlzx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:51:50.332Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"d0fa6d7d-79a8-4834-ba90-16abf4f473de","rayCluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:50.332Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"474abd92-521c-4851-9b2c-8b512c1abc64","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:50.332Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"d0fa6d7d-79a8-4834-ba90-16abf4f473de","pod":"stop-bvlzx-head-cwv4c","ip":"10.133.0.21"}
{"level":"info","ts":"2026-07-13T15:51:50.332Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"d0fa6d7d-79a8-4834-ba90-16abf4f473de","pod":"stop-bvlzx-small-group-worker-r2kxc","ip":"10.134.0.27"}
{"level":"info","ts":"2026-07-13T15:51:50.332Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"2cc73478-74a5-41cd-96b4-42b6cf79a43f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:50.332Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"2cc73478-74a5-41cd-96b4-42b6cf79a43f","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:50.332Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"2cc73478-74a5-41cd-96b4-42b6cf79a43f","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:50.333Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"2cc73478-74a5-41cd-96b4-42b6cf79a43f"}
{"level":"info","ts":"2026-07-13T15:51:50.333Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"2cc73478-74a5-41cd-96b4-42b6cf79a43f","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:51:50.347Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"474abd92-521c-4851-9b2c-8b512c1abc64","name":"stop-bvlzx-head"}
{"level":"info","ts":"2026-07-13T15:51:50.347Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"d0fa6d7d-79a8-4834-ba90-16abf4f473de","certificate":"ray-head-cert-stop-bvlzx","dnsNames":["stop-bvlzx-head-svc","localhost","stop-bvlzx-head-svc.test-ns-5xvz7.svc","stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local"],"ipAddresses":["10.133.0.21","10.134.0.27","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:51:50.375Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"474abd92-521c-4851-9b2c-8b512c1abc64","name":"stop-bvlzx-workers"}
{"level":"info","ts":"2026-07-13T15:51:50.375Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"474abd92-521c-4851-9b2c-8b512c1abc64","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:50.384Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"d0fa6d7d-79a8-4834-ba90-16abf4f473de","certificate":"ray-worker-cert-stop-bvlzx","dnsNames":["stop-bvlzx-worker-svc","localhost","stop-bvlzx-small-group","stop-bvlzx-worker-svc.test-ns-5xvz7.svc","stop-bvlzx-worker-svc.test-ns-5xvz7.svc.cluster.local","*.stop-bvlzx-worker-svc.test-ns-5xvz7.svc","*.stop-bvlzx-worker-svc.test-ns-5xvz7.svc.cluster.local","*-worker-*.test-ns-5xvz7.svc","*-worker-*.test-ns-5xvz7.svc.cluster.local"],"ipAddresses":["10.133.0.21","10.134.0.27","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:51:50.384Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"d0fa6d7d-79a8-4834-ba90-16abf4f473de"}
{"level":"info","ts":"2026-07-13T15:51:52.332Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"b29210bc-d6e0-4458-a747-4d2cd0ccaa74"}
{"level":"info","ts":"2026-07-13T15:51:52.332Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"b29210bc-d6e0-4458-a747-4d2cd0ccaa74"}
{"level":"info","ts":"2026-07-13T15:51:52.332Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"b29210bc-d6e0-4458-a747-4d2cd0ccaa74","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:51:52.332Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"b29210bc-d6e0-4458-a747-4d2cd0ccaa74","Found 1 head Pod":"stop-bvlzx-head-cwv4c","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:51:52.332Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"b29210bc-d6e0-4458-a747-4d2cd0ccaa74","head Pod":"stop-bvlzx-head-cwv4c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-bvlzx-head-cwv4c. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:52.332Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"b29210bc-d6e0-4458-a747-4d2cd0ccaa74","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:51:52.332Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"b29210bc-d6e0-4458-a747-4d2cd0ccaa74","worker Pod":"stop-bvlzx-small-group-worker-r2kxc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-bvlzx-small-group-worker-r2kxc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:51:52.332Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"b29210bc-d6e0-4458-a747-4d2cd0ccaa74","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:52.332Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"b29210bc-d6e0-4458-a747-4d2cd0ccaa74","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:51:52.332Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"b29210bc-d6e0-4458-a747-4d2cd0ccaa74","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:51:52.332Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"b29210bc-d6e0-4458-a747-4d2cd0ccaa74","podName":"stop-bvlzx-head-cwv4c","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:51:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:51:52.333Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"b29210bc-d6e0-4458-a747-4d2cd0ccaa74","podName":"stop-bvlzx-head-cwv4c","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:51:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:51:52.333Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"b29210bc-d6e0-4458-a747-4d2cd0ccaa74","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:51:52.333Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"b29210bc-d6e0-4458-a747-4d2cd0ccaa74","seconds":300}
{"level":"info","ts":"2026-07-13T15:51:52.481Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"0a8fcaec-c370-4fcf-920e-e847e9f9f7af","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:52.481Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"0a8fcaec-c370-4fcf-920e-e847e9f9f7af","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:52.481Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"0a8fcaec-c370-4fcf-920e-e847e9f9f7af","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:52.481Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"0a8fcaec-c370-4fcf-920e-e847e9f9f7af"}
{"level":"info","ts":"2026-07-13T15:51:52.481Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"0a8fcaec-c370-4fcf-920e-e847e9f9f7af","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:51:55.482Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"319ba696-8863-4c60-924d-59aee786654e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:55.482Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"319ba696-8863-4c60-924d-59aee786654e","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:55.482Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"319ba696-8863-4c60-924d-59aee786654e","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:55.482Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"319ba696-8863-4c60-924d-59aee786654e"}
{"level":"info","ts":"2026-07-13T15:51:55.482Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"319ba696-8863-4c60-924d-59aee786654e","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:51:58.483Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"86d84d60-9137-4fef-af81-86fece8a5b47","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:51:58.483Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"86d84d60-9137-4fef-af81-86fece8a5b47","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:51:58.483Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"86d84d60-9137-4fef-af81-86fece8a5b47","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:51:58.484Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"86d84d60-9137-4fef-af81-86fece8a5b47"}
{"level":"info","ts":"2026-07-13T15:51:58.484Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"86d84d60-9137-4fef-af81-86fece8a5b47","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:52:01.351Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"bcabdda2-bb78-4862-9fe5-121358c160a0"}
{"level":"info","ts":"2026-07-13T15:52:01.351Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"bcabdda2-bb78-4862-9fe5-121358c160a0"}
{"level":"info","ts":"2026-07-13T15:52:01.351Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"bcabdda2-bb78-4862-9fe5-121358c160a0","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:52:01.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"bcabdda2-bb78-4862-9fe5-121358c160a0","Found 1 head Pod":"stop-bvlzx-head-cwv4c","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:52:01.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"bcabdda2-bb78-4862-9fe5-121358c160a0","head Pod":"stop-bvlzx-head-cwv4c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-bvlzx-head-cwv4c. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:52:01.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"bcabdda2-bb78-4862-9fe5-121358c160a0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:52:01.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"bcabdda2-bb78-4862-9fe5-121358c160a0","worker Pod":"stop-bvlzx-small-group-worker-r2kxc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-bvlzx-small-group-worker-r2kxc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:52:01.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"bcabdda2-bb78-4862-9fe5-121358c160a0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:52:01.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"bcabdda2-bb78-4862-9fe5-121358c160a0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:52:01.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"bcabdda2-bb78-4862-9fe5-121358c160a0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:52:01.352Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"bcabdda2-bb78-4862-9fe5-121358c160a0","podName":"stop-bvlzx-small-group-worker-r2kxc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:52:01.352Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"bcabdda2-bb78-4862-9fe5-121358c160a0","podName":"stop-bvlzx-small-group-worker-r2kxc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:52:01.352Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"bcabdda2-bb78-4862-9fe5-121358c160a0","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:52:01.352Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"bcabdda2-bb78-4862-9fe5-121358c160a0","name":"stop-bvlzx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:52:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:52:01.364Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"2b651bde-c526-4242-a78a-d9eb086a6cd3","rayCluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:01.364Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"421b35bf-7efc-464a-8b4a-1d9e089374dd","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:01.365Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"2b651bde-c526-4242-a78a-d9eb086a6cd3","pod":"stop-bvlzx-head-cwv4c","ip":"10.133.0.21"}
{"level":"info","ts":"2026-07-13T15:52:01.365Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"2b651bde-c526-4242-a78a-d9eb086a6cd3","pod":"stop-bvlzx-small-group-worker-r2kxc","ip":"10.134.0.27"}
{"level":"info","ts":"2026-07-13T15:52:01.365Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"2b651bde-c526-4242-a78a-d9eb086a6cd3","rayCluster":"stop-bvlzx","podIPs":["10.133.0.21","10.134.0.27"]}
{"level":"info","ts":"2026-07-13T15:52:01.365Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"935c0f4b-1135-40f6-abc6-44717abf1441","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:52:01.365Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"935c0f4b-1135-40f6-abc6-44717abf1441","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:01.365Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"935c0f4b-1135-40f6-abc6-44717abf1441","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:52:01.365Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"935c0f4b-1135-40f6-abc6-44717abf1441"}
{"level":"info","ts":"2026-07-13T15:52:01.365Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"935c0f4b-1135-40f6-abc6-44717abf1441","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:52:01.375Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"421b35bf-7efc-464a-8b4a-1d9e089374dd","name":"stop-bvlzx-head"}
{"level":"info","ts":"2026-07-13T15:52:01.386Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"421b35bf-7efc-464a-8b4a-1d9e089374dd","name":"stop-bvlzx-workers"}
{"level":"info","ts":"2026-07-13T15:52:01.386Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"421b35bf-7efc-464a-8b4a-1d9e089374dd","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:01.484Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"00ce1b55-32e2-494b-86ad-a2b158a16263","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:52:01.484Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"00ce1b55-32e2-494b-86ad-a2b158a16263","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:01.484Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"00ce1b55-32e2-494b-86ad-a2b158a16263","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:52:01.484Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"00ce1b55-32e2-494b-86ad-a2b158a16263"}
{"level":"info","ts":"2026-07-13T15:52:01.484Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"00ce1b55-32e2-494b-86ad-a2b158a16263","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:52:03.364Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06212fcc-e51c-4f87-b89c-6447276a473f"}
{"level":"info","ts":"2026-07-13T15:52:03.364Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06212fcc-e51c-4f87-b89c-6447276a473f"}
{"level":"info","ts":"2026-07-13T15:52:03.364Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06212fcc-e51c-4f87-b89c-6447276a473f","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:52:03.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06212fcc-e51c-4f87-b89c-6447276a473f","Found 1 head Pod":"stop-bvlzx-head-cwv4c","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:52:03.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06212fcc-e51c-4f87-b89c-6447276a473f","head Pod":"stop-bvlzx-head-cwv4c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-bvlzx-head-cwv4c. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:52:03.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06212fcc-e51c-4f87-b89c-6447276a473f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:52:03.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06212fcc-e51c-4f87-b89c-6447276a473f","worker Pod":"stop-bvlzx-small-group-worker-r2kxc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-bvlzx-small-group-worker-r2kxc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:52:03.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06212fcc-e51c-4f87-b89c-6447276a473f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:52:03.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06212fcc-e51c-4f87-b89c-6447276a473f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:52:03.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06212fcc-e51c-4f87-b89c-6447276a473f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:52:03.364Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06212fcc-e51c-4f87-b89c-6447276a473f","podName":"stop-bvlzx-small-group-worker-r2kxc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:52:03.364Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06212fcc-e51c-4f87-b89c-6447276a473f","podName":"stop-bvlzx-small-group-worker-r2kxc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:52:03.364Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06212fcc-e51c-4f87-b89c-6447276a473f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:52:03.365Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"06212fcc-e51c-4f87-b89c-6447276a473f","seconds":300}
{"level":"info","ts":"2026-07-13T15:52:04.485Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b47b758f-68db-4388-95fa-bff86e4cbb6e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:52:04.485Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b47b758f-68db-4388-95fa-bff86e4cbb6e","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:04.485Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b47b758f-68db-4388-95fa-bff86e4cbb6e","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:52:04.485Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b47b758f-68db-4388-95fa-bff86e4cbb6e"}
{"level":"info","ts":"2026-07-13T15:52:04.485Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b47b758f-68db-4388-95fa-bff86e4cbb6e","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:52:07.485Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"065bc4d8-d724-46dd-9328-6194336028d8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:52:07.485Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"065bc4d8-d724-46dd-9328-6194336028d8","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:07.485Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"065bc4d8-d724-46dd-9328-6194336028d8","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:52:07.486Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"065bc4d8-d724-46dd-9328-6194336028d8"}
{"level":"info","ts":"2026-07-13T15:52:07.486Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"065bc4d8-d724-46dd-9328-6194336028d8","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:52:10.487Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"00901fe1-cf32-4534-bfc4-86ae387ec484","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:52:10.487Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"00901fe1-cf32-4534-bfc4-86ae387ec484","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:10.487Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"00901fe1-cf32-4534-bfc4-86ae387ec484","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:52:10.487Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"00901fe1-cf32-4534-bfc4-86ae387ec484"}
{"level":"info","ts":"2026-07-13T15:52:10.487Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"00901fe1-cf32-4534-bfc4-86ae387ec484","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:52:10.928Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"bbb083ac-0833-4a61-9b57-c0cb2573269f","rayCluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:10.928Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"bbb083ac-0833-4a61-9b57-c0cb2573269f","pod":"stop-bvlzx-head-cwv4c","ip":"10.133.0.21"}
{"level":"info","ts":"2026-07-13T15:52:10.928Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"bbb083ac-0833-4a61-9b57-c0cb2573269f","pod":"stop-bvlzx-small-group-worker-r2kxc","ip":"10.134.0.27"}
{"level":"info","ts":"2026-07-13T15:52:10.928Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"bbb083ac-0833-4a61-9b57-c0cb2573269f","rayCluster":"stop-bvlzx","podIPs":["10.133.0.21","10.134.0.27"]}
{"level":"info","ts":"2026-07-13T15:52:13.487Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"eae16007-e53e-4132-860c-1af40849c713","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:52:13.487Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"eae16007-e53e-4132-860c-1af40849c713","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:13.488Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"eae16007-e53e-4132-860c-1af40849c713","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:52:13.488Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"eae16007-e53e-4132-860c-1af40849c713"}
{"level":"info","ts":"2026-07-13T15:52:13.488Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"eae16007-e53e-4132-860c-1af40849c713","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:52:16.488Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"0c5e44d1-46db-47a4-b572-15ee6dc408b1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:52:16.488Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"0c5e44d1-46db-47a4-b572-15ee6dc408b1","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:16.488Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"0c5e44d1-46db-47a4-b572-15ee6dc408b1","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:52:16.488Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"0c5e44d1-46db-47a4-b572-15ee6dc408b1"}
{"level":"info","ts":"2026-07-13T15:52:16.488Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"0c5e44d1-46db-47a4-b572-15ee6dc408b1","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:52:19.489Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"f5c1f8cb-b759-4c3a-8fda-93892e0f9d4e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:52:19.489Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"f5c1f8cb-b759-4c3a-8fda-93892e0f9d4e","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:19.489Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"f5c1f8cb-b759-4c3a-8fda-93892e0f9d4e","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:52:19.489Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"f5c1f8cb-b759-4c3a-8fda-93892e0f9d4e"}
{"level":"info","ts":"2026-07-13T15:52:19.489Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"f5c1f8cb-b759-4c3a-8fda-93892e0f9d4e","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:52:22.490Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"eb560dff-7d5a-4dc5-990d-5ea2064f9bfb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:52:22.490Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"eb560dff-7d5a-4dc5-990d-5ea2064f9bfb","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:22.490Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"eb560dff-7d5a-4dc5-990d-5ea2064f9bfb","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:52:22.490Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"eb560dff-7d5a-4dc5-990d-5ea2064f9bfb"}
{"level":"info","ts":"2026-07-13T15:52:22.490Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"eb560dff-7d5a-4dc5-990d-5ea2064f9bfb","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:52:25.490Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"d9fc7240-c622-4f20-99bc-825a72700aad","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:52:25.491Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"d9fc7240-c622-4f20-99bc-825a72700aad","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:25.491Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"d9fc7240-c622-4f20-99bc-825a72700aad","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:52:25.491Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"d9fc7240-c622-4f20-99bc-825a72700aad"}
{"level":"info","ts":"2026-07-13T15:52:25.491Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"d9fc7240-c622-4f20-99bc-825a72700aad","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:52:28.492Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"1dc26c01-1538-4255-acec-8107ebe0d67b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:52:28.492Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"1dc26c01-1538-4255-acec-8107ebe0d67b","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:28.492Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"1dc26c01-1538-4255-acec-8107ebe0d67b","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:52:28.492Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"1dc26c01-1538-4255-acec-8107ebe0d67b"}
{"level":"info","ts":"2026-07-13T15:52:28.492Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"1dc26c01-1538-4255-acec-8107ebe0d67b","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:52:31.492Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"4aada81d-d00d-4c70-a37d-f7504e6728e4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:52:31.492Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"4aada81d-d00d-4c70-a37d-f7504e6728e4","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:31.492Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"4aada81d-d00d-4c70-a37d-f7504e6728e4","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:52:31.493Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"4aada81d-d00d-4c70-a37d-f7504e6728e4"}
{"level":"info","ts":"2026-07-13T15:52:31.493Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"4aada81d-d00d-4c70-a37d-f7504e6728e4","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:52:34.224Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"a3a54f06-b18b-477d-bb27-04a6f0a1e682","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:52:34.224Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"a3a54f06-b18b-477d-bb27-04a6f0a1e682","pod":"fail-pbfxx-head-27cb9","ip":"10.133.0.20"}
{"level":"info","ts":"2026-07-13T15:52:34.224Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"a3a54f06-b18b-477d-bb27-04a6f0a1e682","pod":"fail-pbfxx-small-group-worker-bhr2g","ip":"10.134.0.26"}
{"level":"info","ts":"2026-07-13T15:52:34.224Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"a3a54f06-b18b-477d-bb27-04a6f0a1e682","rayCluster":"fail-pbfxx","podIPs":["10.133.0.20","10.134.0.26"]}
{"level":"info","ts":"2026-07-13T15:52:34.493Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"8f211a59-23c5-44c5-84e6-99528c0d47e6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:52:34.493Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"8f211a59-23c5-44c5-84e6-99528c0d47e6","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:34.493Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"8f211a59-23c5-44c5-84e6-99528c0d47e6","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:52:34.494Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"8f211a59-23c5-44c5-84e6-99528c0d47e6"}
{"level":"info","ts":"2026-07-13T15:52:34.494Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"8f211a59-23c5-44c5-84e6-99528c0d47e6","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:52:34.936Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"3edb635a-7645-4f98-8b69-764e3c71936a","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:52:34.936Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"3edb635a-7645-4f98-8b69-764e3c71936a","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T15:52:34.936Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"3edb635a-7645-4f98-8b69-764e3c71936a","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T15:52:34.936Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"3edb635a-7645-4f98-8b69-764e3c71936a","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T15:52:37.494Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"9fa0d07c-96eb-44b9-a0bf-6f4046edd248","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:52:37.494Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"9fa0d07c-96eb-44b9-a0bf-6f4046edd248","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:37.494Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"9fa0d07c-96eb-44b9-a0bf-6f4046edd248","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:52:37.494Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"9fa0d07c-96eb-44b9-a0bf-6f4046edd248"}
{"level":"info","ts":"2026-07-13T15:52:37.494Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"9fa0d07c-96eb-44b9-a0bf-6f4046edd248","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:52:40.495Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"7bb18f07-bf5a-42ce-b9fd-4b96d28f24a5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:52:40.495Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"7bb18f07-bf5a-42ce-b9fd-4b96d28f24a5","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:40.495Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"7bb18f07-bf5a-42ce-b9fd-4b96d28f24a5","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:52:40.496Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"7bb18f07-bf5a-42ce-b9fd-4b96d28f24a5"}
{"level":"info","ts":"2026-07-13T15:52:40.496Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"7bb18f07-bf5a-42ce-b9fd-4b96d28f24a5","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:52:43.497Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"d77d7066-e2a7-4470-aef6-fccbb97dfb08","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:52:43.497Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"d77d7066-e2a7-4470-aef6-fccbb97dfb08","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:43.497Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"d77d7066-e2a7-4470-aef6-fccbb97dfb08","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:52:43.497Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"d77d7066-e2a7-4470-aef6-fccbb97dfb08"}
{"level":"info","ts":"2026-07-13T15:52:43.497Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"d77d7066-e2a7-4470-aef6-fccbb97dfb08","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:52:46.498Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"3084d1ea-2154-47c0-b42c-1045bf14314b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:52:46.498Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"3084d1ea-2154-47c0-b42c-1045bf14314b","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:46.498Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"3084d1ea-2154-47c0-b42c-1045bf14314b","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:52:46.498Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"3084d1ea-2154-47c0-b42c-1045bf14314b"}
{"level":"info","ts":"2026-07-13T15:52:46.498Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"3084d1ea-2154-47c0-b42c-1045bf14314b","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:52:49.499Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"7fcbf8b4-ae89-4504-a1a4-69dc15b0626d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:52:49.499Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"7fcbf8b4-ae89-4504-a1a4-69dc15b0626d","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:49.499Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"7fcbf8b4-ae89-4504-a1a4-69dc15b0626d","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:52:49.499Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"7fcbf8b4-ae89-4504-a1a4-69dc15b0626d"}
{"level":"info","ts":"2026-07-13T15:52:49.499Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"7fcbf8b4-ae89-4504-a1a4-69dc15b0626d","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:52:52.500Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"e2b096a9-5fea-46bb-8275-451ea23b9462","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:52:52.500Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"e2b096a9-5fea-46bb-8275-451ea23b9462","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:52.500Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"e2b096a9-5fea-46bb-8275-451ea23b9462","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:52:52.501Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"e2b096a9-5fea-46bb-8275-451ea23b9462"}
{"level":"info","ts":"2026-07-13T15:52:52.501Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"e2b096a9-5fea-46bb-8275-451ea23b9462","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:52:55.501Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b9ffcf67-e0e9-450d-a67c-3c40b6a60b04","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:52:55.501Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b9ffcf67-e0e9-450d-a67c-3c40b6a60b04","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:55.501Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b9ffcf67-e0e9-450d-a67c-3c40b6a60b04","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:52:55.501Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b9ffcf67-e0e9-450d-a67c-3c40b6a60b04"}
{"level":"info","ts":"2026-07-13T15:52:55.501Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b9ffcf67-e0e9-450d-a67c-3c40b6a60b04","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:52:58.502Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b84fdf3e-cbd8-446d-9bae-30af379d5434","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:52:58.502Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b84fdf3e-cbd8-446d-9bae-30af379d5434","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:52:58.502Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b84fdf3e-cbd8-446d-9bae-30af379d5434","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:52:58.502Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b84fdf3e-cbd8-446d-9bae-30af379d5434"}
{"level":"info","ts":"2026-07-13T15:52:58.502Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b84fdf3e-cbd8-446d-9bae-30af379d5434","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:53:01.503Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"1140ff47-b04b-47ee-b8b4-1fc4bfb28d6d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:01.503Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"1140ff47-b04b-47ee-b8b4-1fc4bfb28d6d","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:01.503Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"1140ff47-b04b-47ee-b8b4-1fc4bfb28d6d","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:01.503Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"1140ff47-b04b-47ee-b8b4-1fc4bfb28d6d"}
{"level":"info","ts":"2026-07-13T15:53:01.503Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"1140ff47-b04b-47ee-b8b4-1fc4bfb28d6d","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:53:04.504Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"8f9abdb0-d422-4048-99a0-7ef37d75efa0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:04.504Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"8f9abdb0-d422-4048-99a0-7ef37d75efa0","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:04.504Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"8f9abdb0-d422-4048-99a0-7ef37d75efa0","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:04.504Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"8f9abdb0-d422-4048-99a0-7ef37d75efa0"}
{"level":"info","ts":"2026-07-13T15:53:04.504Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"8f9abdb0-d422-4048-99a0-7ef37d75efa0","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:53:04.735Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"d1b88afb-3200-46f1-ad87-d21667161a02"}
{"level":"info","ts":"2026-07-13T15:53:04.735Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"d1b88afb-3200-46f1-ad87-d21667161a02"}
{"level":"info","ts":"2026-07-13T15:53:04.735Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"d1b88afb-3200-46f1-ad87-d21667161a02","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:04.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"d1b88afb-3200-46f1-ad87-d21667161a02","Found 1 head Pod":"stop-bvlzx-head-cwv4c","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:04.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"d1b88afb-3200-46f1-ad87-d21667161a02","head Pod":"stop-bvlzx-head-cwv4c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-bvlzx-head-cwv4c. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:04.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"d1b88afb-3200-46f1-ad87-d21667161a02","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:04.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"d1b88afb-3200-46f1-ad87-d21667161a02","worker Pod":"stop-bvlzx-small-group-worker-r2kxc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-bvlzx-small-group-worker-r2kxc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:04.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"d1b88afb-3200-46f1-ad87-d21667161a02","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:04.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"d1b88afb-3200-46f1-ad87-d21667161a02","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:53:04.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"d1b88afb-3200-46f1-ad87-d21667161a02","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:04.735Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"d1b88afb-3200-46f1-ad87-d21667161a02","podName":"stop-bvlzx-small-group-worker-r2kxc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:04.735Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"d1b88afb-3200-46f1-ad87-d21667161a02","podName":"stop-bvlzx-small-group-worker-r2kxc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:04.735Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"d1b88afb-3200-46f1-ad87-d21667161a02","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:53:04.735Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"d1b88afb-3200-46f1-ad87-d21667161a02","seconds":300}
{"level":"info","ts":"2026-07-13T15:53:05.738Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5fcdbefc-5c16-430b-acd8-544be334b2ba"}
{"level":"info","ts":"2026-07-13T15:53:05.738Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5fcdbefc-5c16-430b-acd8-544be334b2ba"}
{"level":"info","ts":"2026-07-13T15:53:05.738Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5fcdbefc-5c16-430b-acd8-544be334b2ba","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:05.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5fcdbefc-5c16-430b-acd8-544be334b2ba","Found 1 head Pod":"stop-bvlzx-head-cwv4c","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:05.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5fcdbefc-5c16-430b-acd8-544be334b2ba","head Pod":"stop-bvlzx-head-cwv4c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-bvlzx-head-cwv4c. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:05.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5fcdbefc-5c16-430b-acd8-544be334b2ba","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:05.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5fcdbefc-5c16-430b-acd8-544be334b2ba","worker Pod":"stop-bvlzx-small-group-worker-r2kxc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-bvlzx-small-group-worker-r2kxc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:05.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5fcdbefc-5c16-430b-acd8-544be334b2ba","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:05.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5fcdbefc-5c16-430b-acd8-544be334b2ba","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:53:05.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5fcdbefc-5c16-430b-acd8-544be334b2ba","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:05.738Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5fcdbefc-5c16-430b-acd8-544be334b2ba","podName":"stop-bvlzx-small-group-worker-r2kxc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:05.738Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5fcdbefc-5c16-430b-acd8-544be334b2ba","podName":"stop-bvlzx-small-group-worker-r2kxc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:05.738Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5fcdbefc-5c16-430b-acd8-544be334b2ba","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:53:05.738Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"5fcdbefc-5c16-430b-acd8-544be334b2ba","seconds":300}
{"level":"info","ts":"2026-07-13T15:53:07.505Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"7339dd29-370a-4380-a716-5c7600585d72","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:07.505Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"7339dd29-370a-4380-a716-5c7600585d72","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:07.505Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"7339dd29-370a-4380-a716-5c7600585d72","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:07.505Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"7339dd29-370a-4380-a716-5c7600585d72"}
{"level":"info","ts":"2026-07-13T15:53:07.505Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"7339dd29-370a-4380-a716-5c7600585d72","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:53:10.506Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"08a4b4da-9824-4add-a50e-9b373caea7f6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:10.506Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"08a4b4da-9824-4add-a50e-9b373caea7f6","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:10.506Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"08a4b4da-9824-4add-a50e-9b373caea7f6","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:10.506Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"08a4b4da-9824-4add-a50e-9b373caea7f6"}
{"level":"info","ts":"2026-07-13T15:53:10.506Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"08a4b4da-9824-4add-a50e-9b373caea7f6","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:53:10.929Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"f7f98ece-c680-4504-804e-bcfed5256f31","rayCluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:10.929Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"f7f98ece-c680-4504-804e-bcfed5256f31","pod":"stop-bvlzx-head-cwv4c","ip":"10.133.0.21"}
{"level":"info","ts":"2026-07-13T15:53:10.929Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"f7f98ece-c680-4504-804e-bcfed5256f31","pod":"stop-bvlzx-small-group-worker-r2kxc","ip":"10.134.0.27"}
{"level":"info","ts":"2026-07-13T15:53:10.929Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"f7f98ece-c680-4504-804e-bcfed5256f31","rayCluster":"stop-bvlzx","podIPs":["10.133.0.21","10.134.0.27"]}
{"level":"info","ts":"2026-07-13T15:53:13.507Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"24ea052d-85ae-404b-8f23-4bac586ad79a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:13.507Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"24ea052d-85ae-404b-8f23-4bac586ad79a","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:13.507Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"24ea052d-85ae-404b-8f23-4bac586ad79a","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:13.507Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"24ea052d-85ae-404b-8f23-4bac586ad79a"}
{"level":"info","ts":"2026-07-13T15:53:13.507Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"24ea052d-85ae-404b-8f23-4bac586ad79a","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:53:15.782Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"c10ea389-8ffb-43cc-b53c-eb6bedeb7512"}
{"level":"info","ts":"2026-07-13T15:53:15.782Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"c10ea389-8ffb-43cc-b53c-eb6bedeb7512"}
{"level":"info","ts":"2026-07-13T15:53:15.782Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"c10ea389-8ffb-43cc-b53c-eb6bedeb7512","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:15.782Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"c10ea389-8ffb-43cc-b53c-eb6bedeb7512","Found 1 head Pod":"stop-bvlzx-head-cwv4c","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:15.783Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"c10ea389-8ffb-43cc-b53c-eb6bedeb7512","head Pod":"stop-bvlzx-head-cwv4c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-bvlzx-head-cwv4c. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:15.783Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"c10ea389-8ffb-43cc-b53c-eb6bedeb7512","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:15.783Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"c10ea389-8ffb-43cc-b53c-eb6bedeb7512","worker Pod":"stop-bvlzx-small-group-worker-r2kxc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-bvlzx-small-group-worker-r2kxc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:15.783Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"c10ea389-8ffb-43cc-b53c-eb6bedeb7512","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:15.783Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"c10ea389-8ffb-43cc-b53c-eb6bedeb7512","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:53:15.783Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"c10ea389-8ffb-43cc-b53c-eb6bedeb7512","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:15.783Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"c10ea389-8ffb-43cc-b53c-eb6bedeb7512","podName":"stop-bvlzx-small-group-worker-r2kxc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:15.783Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"c10ea389-8ffb-43cc-b53c-eb6bedeb7512","podName":"stop-bvlzx-small-group-worker-r2kxc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:15.783Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"c10ea389-8ffb-43cc-b53c-eb6bedeb7512","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:53:15.783Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"c10ea389-8ffb-43cc-b53c-eb6bedeb7512","seconds":300}
{"level":"info","ts":"2026-07-13T15:53:16.508Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"3bfd54f3-a673-42a7-bd7b-c0a71034c8e6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:16.508Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"3bfd54f3-a673-42a7-bd7b-c0a71034c8e6","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:16.508Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"3bfd54f3-a673-42a7-bd7b-c0a71034c8e6","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:16.508Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"3bfd54f3-a673-42a7-bd7b-c0a71034c8e6"}
{"level":"info","ts":"2026-07-13T15:53:16.508Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"3bfd54f3-a673-42a7-bd7b-c0a71034c8e6","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:53:16.788Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2130901a-8f27-40db-9e1b-f28a0a7c866b"}
{"level":"info","ts":"2026-07-13T15:53:16.788Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2130901a-8f27-40db-9e1b-f28a0a7c866b"}
{"level":"info","ts":"2026-07-13T15:53:16.788Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2130901a-8f27-40db-9e1b-f28a0a7c866b","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:16.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2130901a-8f27-40db-9e1b-f28a0a7c866b","Found 1 head Pod":"stop-bvlzx-head-cwv4c","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:16.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2130901a-8f27-40db-9e1b-f28a0a7c866b","head Pod":"stop-bvlzx-head-cwv4c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-bvlzx-head-cwv4c. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:16.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2130901a-8f27-40db-9e1b-f28a0a7c866b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:16.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2130901a-8f27-40db-9e1b-f28a0a7c866b","worker Pod":"stop-bvlzx-small-group-worker-r2kxc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-bvlzx-small-group-worker-r2kxc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:16.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2130901a-8f27-40db-9e1b-f28a0a7c866b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:16.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2130901a-8f27-40db-9e1b-f28a0a7c866b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:53:16.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2130901a-8f27-40db-9e1b-f28a0a7c866b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:16.788Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2130901a-8f27-40db-9e1b-f28a0a7c866b","podName":"stop-bvlzx-small-group-worker-r2kxc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:51:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:53:16.789Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2130901a-8f27-40db-9e1b-f28a0a7c866b","podName":"stop-bvlzx-small-group-worker-r2kxc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:51:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:53:16.789Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2130901a-8f27-40db-9e1b-f28a0a7c866b","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:53:16.789Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"2130901a-8f27-40db-9e1b-f28a0a7c866b","name":"stop-bvlzx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:52:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:53:16.800Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"88e2cf81-154e-41e1-a683-d745a6bf4cb1","rayCluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:16.800Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"75c82f44-269a-499b-bc32-7240ca63e99d","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:16.800Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"88e2cf81-154e-41e1-a683-d745a6bf4cb1","pod":"stop-bvlzx-head-cwv4c","ip":"10.133.0.21"}
{"level":"info","ts":"2026-07-13T15:53:16.800Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"88e2cf81-154e-41e1-a683-d745a6bf4cb1","pod":"stop-bvlzx-small-group-worker-r2kxc","ip":"10.134.0.27"}
{"level":"info","ts":"2026-07-13T15:53:16.800Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b4ab9eae-b33d-4cd2-8a5a-df315f3da557","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:16.800Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b4ab9eae-b33d-4cd2-8a5a-df315f3da557","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:16.800Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"88e2cf81-154e-41e1-a683-d745a6bf4cb1","rayCluster":"stop-bvlzx","podIPs":["10.133.0.21","10.134.0.27"]}
{"level":"info","ts":"2026-07-13T15:53:16.800Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b4ab9eae-b33d-4cd2-8a5a-df315f3da557","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:16.800Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b4ab9eae-b33d-4cd2-8a5a-df315f3da557"}
{"level":"info","ts":"2026-07-13T15:53:16.800Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b4ab9eae-b33d-4cd2-8a5a-df315f3da557","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:53:16.815Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"75c82f44-269a-499b-bc32-7240ca63e99d","name":"stop-bvlzx-head"}
{"level":"info","ts":"2026-07-13T15:53:16.824Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"75c82f44-269a-499b-bc32-7240ca63e99d","name":"stop-bvlzx-workers"}
{"level":"info","ts":"2026-07-13T15:53:16.824Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"75c82f44-269a-499b-bc32-7240ca63e99d","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:18.800Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"340e7fd4-ba1b-4625-8023-37bfa02a009f"}
{"level":"info","ts":"2026-07-13T15:53:18.800Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"340e7fd4-ba1b-4625-8023-37bfa02a009f"}
{"level":"info","ts":"2026-07-13T15:53:18.800Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"340e7fd4-ba1b-4625-8023-37bfa02a009f","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:18.800Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"340e7fd4-ba1b-4625-8023-37bfa02a009f","Found 1 head Pod":"stop-bvlzx-head-cwv4c","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:18.800Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"340e7fd4-ba1b-4625-8023-37bfa02a009f","head Pod":"stop-bvlzx-head-cwv4c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-bvlzx-head-cwv4c. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:18.800Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"340e7fd4-ba1b-4625-8023-37bfa02a009f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:18.800Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"340e7fd4-ba1b-4625-8023-37bfa02a009f","worker Pod":"stop-bvlzx-small-group-worker-r2kxc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-bvlzx-small-group-worker-r2kxc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:18.800Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"340e7fd4-ba1b-4625-8023-37bfa02a009f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:18.800Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"340e7fd4-ba1b-4625-8023-37bfa02a009f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:53:18.800Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"340e7fd4-ba1b-4625-8023-37bfa02a009f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:18.800Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"340e7fd4-ba1b-4625-8023-37bfa02a009f","podName":"stop-bvlzx-small-group-worker-r2kxc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:51:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:53:18.800Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"340e7fd4-ba1b-4625-8023-37bfa02a009f","podName":"stop-bvlzx-small-group-worker-r2kxc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:51:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:53:18.801Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"340e7fd4-ba1b-4625-8023-37bfa02a009f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:53:18.801Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"340e7fd4-ba1b-4625-8023-37bfa02a009f","seconds":300}
{"level":"info","ts":"2026-07-13T15:53:19.508Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"a516a768-a45d-4490-9f71-1cd1b6fc3f27","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:19.508Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"a516a768-a45d-4490-9f71-1cd1b6fc3f27","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:19.508Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"a516a768-a45d-4490-9f71-1cd1b6fc3f27","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:19.508Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"a516a768-a45d-4490-9f71-1cd1b6fc3f27"}
{"level":"info","ts":"2026-07-13T15:53:19.508Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"a516a768-a45d-4490-9f71-1cd1b6fc3f27","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:53:22.510Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"edaa7fed-0777-46f9-b8b5-06064a0e61f5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:22.510Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"edaa7fed-0777-46f9-b8b5-06064a0e61f5","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:22.510Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"edaa7fed-0777-46f9-b8b5-06064a0e61f5","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:22.510Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"edaa7fed-0777-46f9-b8b5-06064a0e61f5"}
{"level":"info","ts":"2026-07-13T15:53:22.510Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"edaa7fed-0777-46f9-b8b5-06064a0e61f5","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:53:25.510Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"d39229a1-7b7f-4af3-9413-6b45fb7e8bba","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:25.510Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"d39229a1-7b7f-4af3-9413-6b45fb7e8bba","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:25.510Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"d39229a1-7b7f-4af3-9413-6b45fb7e8bba","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:25.511Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"d39229a1-7b7f-4af3-9413-6b45fb7e8bba"}
{"level":"info","ts":"2026-07-13T15:53:25.511Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"d39229a1-7b7f-4af3-9413-6b45fb7e8bba","RayCluster":"stop-bvlzx","State":""}
{"level":"info","ts":"2026-07-13T15:53:27.811Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"71c01851-8f8f-435e-b47b-2d1a71b3e866"}
{"level":"info","ts":"2026-07-13T15:53:27.812Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"71c01851-8f8f-435e-b47b-2d1a71b3e866"}
{"level":"info","ts":"2026-07-13T15:53:27.812Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"71c01851-8f8f-435e-b47b-2d1a71b3e866","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:27.812Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"71c01851-8f8f-435e-b47b-2d1a71b3e866","Found 1 head Pod":"stop-bvlzx-head-cwv4c","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:27.812Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"71c01851-8f8f-435e-b47b-2d1a71b3e866","head Pod":"stop-bvlzx-head-cwv4c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-bvlzx-head-cwv4c. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:27.812Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"71c01851-8f8f-435e-b47b-2d1a71b3e866","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:27.812Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"71c01851-8f8f-435e-b47b-2d1a71b3e866","worker Pod":"stop-bvlzx-small-group-worker-r2kxc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-bvlzx-small-group-worker-r2kxc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:27.812Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"71c01851-8f8f-435e-b47b-2d1a71b3e866","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:27.812Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"71c01851-8f8f-435e-b47b-2d1a71b3e866","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:53:27.812Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"71c01851-8f8f-435e-b47b-2d1a71b3e866","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:27.812Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"71c01851-8f8f-435e-b47b-2d1a71b3e866","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-13T15:53:27.812Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"71c01851-8f8f-435e-b47b-2d1a71b3e866","name":"stop-bvlzx","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:53:27.823Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"4662ba0d-c113-4a6a-875f-51f2ff8cade8","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:27.823Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"2c4e6d6e-a280-46fd-9fc4-88f8cbd9106a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:27.823Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"2c4e6d6e-a280-46fd-9fc4-88f8cbd9106a","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:27.823Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"57a3df69-5184-44e5-a7f7-450d12523b24","rayCluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:27.824Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"2c4e6d6e-a280-46fd-9fc4-88f8cbd9106a","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:27.824Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"2c4e6d6e-a280-46fd-9fc4-88f8cbd9106a"}
{"level":"info","ts":"2026-07-13T15:53:27.824Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"2c4e6d6e-a280-46fd-9fc4-88f8cbd9106a","head service name":"stop-bvlzx-head-svc","namespace":"test-ns-5xvz7"}
{"level":"info","ts":"2026-07-13T15:53:27.824Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"2c4e6d6e-a280-46fd-9fc4-88f8cbd9106a","head service URL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-13T15:53:27.824Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"2c4e6d6e-a280-46fd-9fc4-88f8cbd9106a","SubmissionMode":"HTTPMode","RayCluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:27.824Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"2c4e6d6e-a280-46fd-9fc4-88f8cbd9106a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:53:27.824Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"2c4e6d6e-a280-46fd-9fc4-88f8cbd9106a","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T15:53:27.824Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"57a3df69-5184-44e5-a7f7-450d12523b24","pod":"stop-bvlzx-head-cwv4c","ip":"10.133.0.21"}
{"level":"info","ts":"2026-07-13T15:53:27.824Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"57a3df69-5184-44e5-a7f7-450d12523b24","pod":"stop-bvlzx-small-group-worker-r2kxc","ip":"10.134.0.27"}
{"level":"info","ts":"2026-07-13T15:53:27.824Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"57a3df69-5184-44e5-a7f7-450d12523b24","rayCluster":"stop-bvlzx","podIPs":["10.133.0.21","10.134.0.27"]}
{"level":"info","ts":"2026-07-13T15:53:27.922Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"f49f0c77-e414-47d2-acd9-b0162465505d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:27.922Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"f49f0c77-e414-47d2-acd9-b0162465505d","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:27.922Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"f49f0c77-e414-47d2-acd9-b0162465505d","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:27.922Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"f49f0c77-e414-47d2-acd9-b0162465505d"}
{"level":"info","ts":"2026-07-13T15:53:28.021Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"f49f0c77-e414-47d2-acd9-b0162465505d","JobId":"stop-km8dj"}
{"level":"info","ts":"2026-07-13T15:53:28.021Z","logger":"controllers.RayJob","msg":"Submit a ray job","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"f49f0c77-e414-47d2-acd9-b0162465505d","rayJob":"stop","jobInfo":"{\"entrypoint\":\"python /home/ray/jobs/stop.py\",\"submission_id\":\"stop-km8dj\"}"}
{"level":"info","ts":"2026-07-13T15:53:28.021Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"4662ba0d-c113-4a6a-875f-51f2ff8cade8","name":"stop-bvlzx-head"}
{"level":"info","ts":"2026-07-13T15:53:28.030Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"4662ba0d-c113-4a6a-875f-51f2ff8cade8","name":"stop-bvlzx-workers"}
{"level":"info","ts":"2026-07-13T15:53:28.030Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"4662ba0d-c113-4a6a-875f-51f2ff8cade8","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:28.858Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"5014d8a3-67f3-4914-82e3-cbc281112ff2","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:28.858Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"5014d8a3-67f3-4914-82e3-cbc281112ff2","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:28.858Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"5014d8a3-67f3-4914-82e3-cbc281112ff2","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:28.858Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"5014d8a3-67f3-4914-82e3-cbc281112ff2"}
{"level":"info","ts":"2026-07-13T15:53:28.861Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"5014d8a3-67f3-4914-82e3-cbc281112ff2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:53:28Z"},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:53:28.861Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"5014d8a3-67f3-4914-82e3-cbc281112ff2","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T15:53:28.874Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"4b337016-3ef4-4e37-a372-9d8c0bd7e456","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:28.874Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"4b337016-3ef4-4e37-a372-9d8c0bd7e456","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:28.875Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"4b337016-3ef4-4e37-a372-9d8c0bd7e456","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:28.875Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"4b337016-3ef4-4e37-a372-9d8c0bd7e456"}
{"level":"info","ts":"2026-07-13T15:53:28.877Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"4b337016-3ef4-4e37-a372-9d8c0bd7e456","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:53:28Z"},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:53:28Z"},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:53:29.823Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3c394823-31b5-48b7-933e-5cb8ab42c8dd"}
{"level":"info","ts":"2026-07-13T15:53:29.823Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3c394823-31b5-48b7-933e-5cb8ab42c8dd"}
{"level":"info","ts":"2026-07-13T15:53:29.823Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3c394823-31b5-48b7-933e-5cb8ab42c8dd","1 head service found":"stop-bvlzx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:29.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3c394823-31b5-48b7-933e-5cb8ab42c8dd","Found 1 head Pod":"stop-bvlzx-head-cwv4c","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:29.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3c394823-31b5-48b7-933e-5cb8ab42c8dd","head Pod":"stop-bvlzx-head-cwv4c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-bvlzx-head-cwv4c. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:29.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3c394823-31b5-48b7-933e-5cb8ab42c8dd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:29.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3c394823-31b5-48b7-933e-5cb8ab42c8dd","worker Pod":"stop-bvlzx-small-group-worker-r2kxc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-bvlzx-small-group-worker-r2kxc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:29.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3c394823-31b5-48b7-933e-5cb8ab42c8dd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:29.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3c394823-31b5-48b7-933e-5cb8ab42c8dd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:53:29.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3c394823-31b5-48b7-933e-5cb8ab42c8dd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:29.824Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3c394823-31b5-48b7-933e-5cb8ab42c8dd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:53:29.824Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"3c394823-31b5-48b7-933e-5cb8ab42c8dd","seconds":300}
{"level":"info","ts":"2026-07-13T15:53:31.859Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"88988d24-8f7f-4695-b09a-37c1cf3f7787","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:31.859Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"88988d24-8f7f-4695-b09a-37c1cf3f7787","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:31.859Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"88988d24-8f7f-4695-b09a-37c1cf3f7787","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:31.859Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"88988d24-8f7f-4695-b09a-37c1cf3f7787"}
{"level":"info","ts":"2026-07-13T15:53:31.862Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"88988d24-8f7f-4695-b09a-37c1cf3f7787","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:53:28Z"},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:53:28Z"},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:53:31.862Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"88988d24-8f7f-4695-b09a-37c1cf3f7787","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T15:53:31.874Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"068a32d6-b3d4-4b87-a36d-7608c0dc813e","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:31.874Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"068a32d6-b3d4-4b87-a36d-7608c0dc813e","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:31.874Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"068a32d6-b3d4-4b87-a36d-7608c0dc813e","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:31.874Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"068a32d6-b3d4-4b87-a36d-7608c0dc813e"}
{"level":"info","ts":"2026-07-13T15:53:31.877Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"068a32d6-b3d4-4b87-a36d-7608c0dc813e","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:53:28Z"},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:53:28Z"},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:53:34.225Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c5c373af-b5e8-4fa1-8690-d60715a651eb","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:53:34.225Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c5c373af-b5e8-4fa1-8690-d60715a651eb","pod":"fail-pbfxx-head-27cb9","ip":"10.133.0.20"}
{"level":"info","ts":"2026-07-13T15:53:34.225Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c5c373af-b5e8-4fa1-8690-d60715a651eb","pod":"fail-pbfxx-small-group-worker-bhr2g","ip":"10.134.0.26"}
{"level":"info","ts":"2026-07-13T15:53:34.225Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c5c373af-b5e8-4fa1-8690-d60715a651eb","rayCluster":"fail-pbfxx","podIPs":["10.133.0.20","10.134.0.26"]}
{"level":"info","ts":"2026-07-13T15:53:34.875Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"f9777b62-d1ef-43ea-9258-737ffd9f53a4","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:34.875Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"f9777b62-d1ef-43ea-9258-737ffd9f53a4","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:34.875Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"f9777b62-d1ef-43ea-9258-737ffd9f53a4","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:34.875Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"f9777b62-d1ef-43ea-9258-737ffd9f53a4"}
{"level":"info","ts":"2026-07-13T15:53:34.877Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"f9777b62-d1ef-43ea-9258-737ffd9f53a4","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:53:28Z"},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:53:28Z"},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:53:34.936Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"74dc4cfa-616b-47ed-936a-4c3cdaa8191f","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:53:34.936Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"74dc4cfa-616b-47ed-936a-4c3cdaa8191f","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T15:53:34.936Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"74dc4cfa-616b-47ed-936a-4c3cdaa8191f","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T15:53:34.936Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"74dc4cfa-616b-47ed-936a-4c3cdaa8191f","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T15:53:37.878Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"928f7665-0411-45fd-a5cc-1d9141574cae","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:37.878Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"928f7665-0411-45fd-a5cc-1d9141574cae","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:37.878Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"928f7665-0411-45fd-a5cc-1d9141574cae","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:37.878Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"928f7665-0411-45fd-a5cc-1d9141574cae"}
{"level":"info","ts":"2026-07-13T15:53:37.881Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"928f7665-0411-45fd-a5cc-1d9141574cae","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:53:28Z"},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:53:28Z"},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:53:40.882Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"eef510a4-98b3-4dba-9c6f-da7c0fe956ac","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:40.882Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"eef510a4-98b3-4dba-9c6f-da7c0fe956ac","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:40.882Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"eef510a4-98b3-4dba-9c6f-da7c0fe956ac","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:40.882Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"eef510a4-98b3-4dba-9c6f-da7c0fe956ac"}
{"level":"info","ts":"2026-07-13T15:53:40.884Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"eef510a4-98b3-4dba-9c6f-da7c0fe956ac","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:53:28Z"},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:53:28Z"},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:53:43.885Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"c9d99cc9-e332-4146-823e-11f5c09b63db","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:43.885Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"c9d99cc9-e332-4146-823e-11f5c09b63db","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:43.885Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"c9d99cc9-e332-4146-823e-11f5c09b63db","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:43.885Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"c9d99cc9-e332-4146-823e-11f5c09b63db"}
{"level":"info","ts":"2026-07-13T15:53:43.888Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"c9d99cc9-e332-4146-823e-11f5c09b63db","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:53:28Z"},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:53:28Z"},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:53:46.889Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"88abd6b1-f36e-419e-8446-ffa0fa41442e","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:46.889Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"88abd6b1-f36e-419e-8446-ffa0fa41442e","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:46.889Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"88abd6b1-f36e-419e-8446-ffa0fa41442e","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:46.889Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"88abd6b1-f36e-419e-8446-ffa0fa41442e"}
{"level":"info","ts":"2026-07-13T15:53:46.892Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"88abd6b1-f36e-419e-8446-ffa0fa41442e","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:53:28Z"},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:53:28Z"},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:53:49.893Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"11778c29-211e-4776-bf6f-bd4e148bb5d2","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:49.893Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"11778c29-211e-4776-bf6f-bd4e148bb5d2","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:49.893Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"11778c29-211e-4776-bf6f-bd4e148bb5d2","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:49.893Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"11778c29-211e-4776-bf6f-bd4e148bb5d2"}
{"level":"info","ts":"2026-07-13T15:53:49.896Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"11778c29-211e-4776-bf6f-bd4e148bb5d2","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:53:28Z"},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:53:28Z"},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:53:52.896Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b777c2b7-3ece-4ee4-9879-a4b48f61ee81","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:52.896Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b777c2b7-3ece-4ee4-9879-a4b48f61ee81","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:52.896Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b777c2b7-3ece-4ee4-9879-a4b48f61ee81","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:52.896Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b777c2b7-3ece-4ee4-9879-a4b48f61ee81"}
{"level":"info","ts":"2026-07-13T15:53:52.899Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b777c2b7-3ece-4ee4-9879-a4b48f61ee81","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:53:28Z"},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:53:28Z","endTime":"2026-07-13T15:53:50Z"},"jobId":"stop-km8dj","rayClusterName":"stop-bvlzx","dashboardURL":"stop-bvlzx-head-svc.test-ns-5xvz7.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Complete","message":"Job was intentionally stopped.","startTime":"2026-07-13T15:51:40Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:27Z","stateTransitionTimes":{"ready":"2026-07-13T15:53:27Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.21","serviceIP":"10.133.0.21","podName":"stop-bvlzx-head-cwv4c","serviceName":"stop-bvlzx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:52:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:53:27Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:51:40Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:51:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:53:52.899Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"b777c2b7-3ece-4ee4-9879-a4b48f61ee81","old JobStatus":"RUNNING","new JobStatus":"STOPPED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-07-13T15:53:52.916Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"29ac860e-b146-4796-a6e8-8cebb7c5e1c5","JobStatus":"STOPPED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:52.916Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"29ac860e-b146-4796-a6e8-8cebb7c5e1c5","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 15:53:52 +0000 UTC","Now":"2026-07-13T15:53:52.916Z","ShutdownTime":"2026-07-13T15:53:52.000Z"}
{"level":"info","ts":"2026-07-13T15:53:53.096Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"aff2556c-4bc6-46b4-8b06-59693e97dddf","DeletionTimestamp":"2026-07-13 15:53:53 +0000 UTC"}
{"level":"info","ts":"2026-07-13T15:53:53.096Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"aff2556c-4bc6-46b4-8b06-59693e97dddf","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T15:53:53.107Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"ce542204-3f82-4204-85a1-0e1e4b9e51d3"}
{"level":"info","ts":"2026-07-13T15:53:53.125Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"a735bc16-812a-45c0-9057-33eba4c2d0c1","namespacedName":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:53.125Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"a735bc16-812a-45c0-9057-33eba4c2d0c1","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:53.125Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"51a8f7e1-66a6-4d4a-ab8a-74563914adcf","rayCluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:53.125Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"reconcileID":"9eecbc74-d07b-411e-ba8a-1345d8575dcf"}
{"level":"info","ts":"2026-07-13T15:53:53.125Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"16d5a0c1-b5a3-44a7-be2e-76c4d8318a26"}
{"level":"info","ts":"2026-07-13T15:53:53.125Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"a735bc16-812a-45c0-9057-33eba4c2d0c1","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:53.125Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"29f63149-f25b-47ba-8925-a6412706ecd0"}
{"level":"info","ts":"2026-07-13T15:53:53.125Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"51a8f7e1-66a6-4d4a-ab8a-74563914adcf","pod":"stop-bvlzx-head-cwv4c","ip":"10.133.0.21"}
{"level":"info","ts":"2026-07-13T15:53:53.125Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"a735bc16-812a-45c0-9057-33eba4c2d0c1","namespace":"test-ns-5xvz7","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-07-13T15:53:53.125Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"51a8f7e1-66a6-4d4a-ab8a-74563914adcf","pod":"stop-bvlzx-small-group-worker-r2kxc","ip":"10.134.0.27"}
{"level":"info","ts":"2026-07-13T15:53:53.125Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"a735bc16-812a-45c0-9057-33eba4c2d0c1","name":"kuberay-gateway-access","namespace":"test-ns-5xvz7","remainingClusters":1}
{"level":"info","ts":"2026-07-13T15:53:53.125Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"51a8f7e1-66a6-4d4a-ab8a-74563914adcf","rayCluster":"stop-bvlzx","podIPs":["10.133.0.21","10.134.0.27"]}
{"level":"info","ts":"2026-07-13T15:53:53.130Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"a735bc16-812a-45c0-9057-33eba4c2d0c1","name":"test-ns-5xvz7-stop-bvlzx","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-13T15:53:53.134Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"a735bc16-812a-45c0-9057-33eba4c2d0c1","configMap":"kube-rbac-proxy-config-stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:53.139Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"a735bc16-812a-45c0-9057-33eba4c2d0c1","serviceAccount":"stop-bvlzx-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:53:53.158Z","logger":"raycluster-resource","msg":"default","name":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:53.158Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-bvlzx","namespace":"test-ns-5xvz7"}
{"level":"info","ts":"2026-07-13T15:53:53.167Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"a735bc16-812a-45c0-9057-33eba4c2d0c1","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:53.167Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"5388bb7a-fbfd-43c2-bf8e-9dc8195b5a0c","namespacedName":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:53.167Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"5388bb7a-fbfd-43c2-bf8e-9dc8195b5a0c","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:53.167Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"5388bb7a-fbfd-43c2-bf8e-9dc8195b5a0c","cluster":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:53.167Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"5388bb7a-fbfd-43c2-bf8e-9dc8195b5a0c","namespace":"test-ns-5xvz7","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-07-13T15:53:53.167Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"5388bb7a-fbfd-43c2-bf8e-9dc8195b5a0c","name":"kuberay-gateway-access","namespace":"test-ns-5xvz7","remainingClusters":1}
{"level":"info","ts":"2026-07-13T15:53:53.167Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"762dd834-d867-4108-84f1-c74e50652c19","namespace":"test-ns-5xvz7","clusterName":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:53.167Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"c8ce1360-a38a-4d11-a6dd-5a9e9696810c"}
{"level":"info","ts":"2026-07-13T15:53:53.167Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"05e66de0-ea18-4f58-ab60-29b52c144e85"}
{"level":"error","ts":"2026-07-13T15:53:53.171Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"5388bb7a-fbfd-43c2-bf8e-9dc8195b5a0c","error":"rayclusters.ray.io \"stop-bvlzx\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.171Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"5388bb7a-fbfd-43c2-bf8e-9dc8195b5a0c"}
{"level":"error","ts":"2026-07-13T15:53:53.171Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"5388bb7a-fbfd-43c2-bf8e-9dc8195b5a0c","error":"rayclusters.ray.io \"stop-bvlzx\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.171Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"7cce48c0-8bb6-45d2-b0d7-a9dcd1fb7fbd","namespacedName":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:53.171Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"7cce48c0-8bb6-45d2-b0d7-a9dcd1fb7fbd"}
{"level":"info","ts":"2026-07-13T15:53:53.171Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"7cce48c0-8bb6-45d2-b0d7-a9dcd1fb7fbd","namespace":"test-ns-5xvz7","authClusters":1}
{"level":"info","ts":"2026-07-13T15:53:53.171Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"7cce48c0-8bb6-45d2-b0d7-a9dcd1fb7fbd","namespace":"test-ns-5xvz7","authClusters":1}
{"level":"info","ts":"2026-07-13T15:53:53.172Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"64bfde0d-ea59-46da-b292-f66f73a7d9a5","namespacedName":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:53.172Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"64bfde0d-ea59-46da-b292-f66f73a7d9a5"}
{"level":"info","ts":"2026-07-13T15:53:53.172Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"64bfde0d-ea59-46da-b292-f66f73a7d9a5","namespace":"test-ns-5xvz7","authClusters":1}
{"level":"info","ts":"2026-07-13T15:53:53.172Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"64bfde0d-ea59-46da-b292-f66f73a7d9a5","namespace":"test-ns-5xvz7","authClusters":1}
{"level":"info","ts":"2026-07-13T15:53:53.191Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"3b651a16-051b-4966-bb81-94b011a87c46","namespacedName":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:53:53.221Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"3b651a16-051b-4966-bb81-94b011a87c46"}
{"level":"info","ts":"2026-07-13T15:53:53.221Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"3b651a16-051b-4966-bb81-94b011a87c46","namespace":"test-ns-5xvz7","authClusters":1}
{"level":"info","ts":"2026-07-13T15:53:53.221Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"3b651a16-051b-4966-bb81-94b011a87c46","namespace":"test-ns-5xvz7","authClusters":1}
{"level":"info","ts":"2026-07-13T15:53:53.221Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"146d8701-45b6-4ccb-a0bb-8acce90ef817"}
{"level":"info","ts":"2026-07-13T15:53:53.222Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"74deced7-52c4-4208-a9a5-6705fc9b0b0d"}
{"level":"error","ts":"2026-07-13T15:53:53.222Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"762dd834-d867-4108-84f1-c74e50652c19","issuer":"ray-selfsigned-issuer-stop-bvlzx","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-stop-bvlzx\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.225Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8fcac604-59a4-4408-9dca-0b93ef2f544d","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:53:53.225Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8fcac604-59a4-4408-9dca-0b93ef2f544d","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T15:53:53.241Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8fcac604-59a4-4408-9dca-0b93ef2f544d"}
{"level":"info","ts":"2026-07-13T15:53:53.241Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8fcac604-59a4-4408-9dca-0b93ef2f544d","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-13T15:53:53.241Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8fcac604-59a4-4408-9dca-0b93ef2f544d","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-wmk2z","rayClusterName":"long-running-qtcvg","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:53:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:53:53.242Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8fcac604-59a4-4408-9dca-0b93ef2f544d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T15:53:53.255Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"762dd834-d867-4108-84f1-c74e50652c19","errorCount":1}
{"level":"info","ts":"2026-07-13T15:53:53.255Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"762dd834-d867-4108-84f1-c74e50652c19"}
{"level":"error","ts":"2026-07-13T15:53:53.255Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"762dd834-d867-4108-84f1-c74e50652c19","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-stop-bvlzx\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.255Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7b8e45af-01cb-4d2e-b2b0-2fc40dd2a420","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:53:53.255Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7b8e45af-01cb-4d2e-b2b0-2fc40dd2a420","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.255Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7b8e45af-01cb-4d2e-b2b0-2fc40dd2a420","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:53:53.255Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7b8e45af-01cb-4d2e-b2b0-2fc40dd2a420","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:53:53.260Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"216cc3c4-d7ba-4331-9c33-662ece21f8fa","namespace":"test-ns-5xvz7","clusterName":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:53.260Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-bvlzx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"stop-bvlzx","reconcileID":"216cc3c4-d7ba-4331-9c33-662ece21f8fa","namespace":"test-ns-5xvz7","clusterName":"stop-bvlzx"}
{"level":"info","ts":"2026-07-13T15:53:53.321Z","logger":"raycluster-resource","msg":"default","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.321Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-qtcvg","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:53:53.337Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7b8e45af-01cb-4d2e-b2b0-2fc40dd2a420","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:53:53.338Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4e652317-0e37-4b89-8796-dfcde35e3f93"}
{"level":"info","ts":"2026-07-13T15:53:53.338Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4e652317-0e37-4b89-8796-dfcde35e3f93"}
{"level":"info","ts":"2026-07-13T15:53:53.338Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7b8e45af-01cb-4d2e-b2b0-2fc40dd2a420"}
{"level":"info","ts":"2026-07-13T15:53:53.338Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7b8e45af-01cb-4d2e-b2b0-2fc40dd2a420","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:53:53.338Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"eafd15cf-1764-480b-87ea-59a0ace180f0","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.338Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"44f99fa7-e8bb-4066-a8ed-8d1e753e6729","namespacedName":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:53:53.338Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"44f99fa7-e8bb-4066-a8ed-8d1e753e6729","mode":"IntegratedOAuth","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.338Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"44f99fa7-e8bb-4066-a8ed-8d1e753e6729","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.338Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"44f99fa7-e8bb-4066-a8ed-8d1e753e6729","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.338Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2a5168bc-97f8-437c-9085-0318c034df02","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:53:53.338Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2a5168bc-97f8-437c-9085-0318c034df02","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.338Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2a5168bc-97f8-437c-9085-0318c034df02","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:53:53.338Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"97045da1-0194-4a17-a57a-e23c07535bfc","rayCluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.338Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"97045da1-0194-4a17-a57a-e23c07535bfc","rayCluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.338Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2a5168bc-97f8-437c-9085-0318c034df02"}
{"level":"info","ts":"2026-07-13T15:53:53.338Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2a5168bc-97f8-437c-9085-0318c034df02","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:53:53.343Z","logger":"raycluster-resource","msg":"default","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.343Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-qtcvg","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:53:53.345Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4e652317-0e37-4b89-8796-dfcde35e3f93","name":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:53.345Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4e652317-0e37-4b89-8796-dfcde35e3f93","error":"head certificate secret ray-head-secret-long-running-qtcvg not found"}
{"level":"error","ts":"2026-07-13T15:53:53.345Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4e652317-0e37-4b89-8796-dfcde35e3f93","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.345Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4e652317-0e37-4b89-8796-dfcde35e3f93","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found"}
{"level":"info","ts":"2026-07-13T15:53:53.345Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4e652317-0e37-4b89-8796-dfcde35e3f93","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:53.346Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4e652317-0e37-4b89-8796-dfcde35e3f93","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:53.346Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4e652317-0e37-4b89-8796-dfcde35e3f93","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:53.346Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4e652317-0e37-4b89-8796-dfcde35e3f93","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:53:53.346Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4e652317-0e37-4b89-8796-dfcde35e3f93","name":"long-running-qtcvg","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-qtcvg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:53:53.352Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"e1bbab1b-d236-4506-a3d5-9c78a3e1efeb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:53:53.352Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"e1bbab1b-d236-4506-a3d5-9c78a3e1efeb","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.352Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"e1bbab1b-d236-4506-a3d5-9c78a3e1efeb","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:53:53.352Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"e1bbab1b-d236-4506-a3d5-9c78a3e1efeb"}
{"level":"info","ts":"2026-07-13T15:53:53.352Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"e1bbab1b-d236-4506-a3d5-9c78a3e1efeb","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:53:53.355Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"eafd15cf-1764-480b-87ea-59a0ace180f0","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.355Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"e50eecda-9c69-4ba9-9166-d306aebf3ed1","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.421Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4e652317-0e37-4b89-8796-dfcde35e3f93","name":"long-running-qtcvg","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qtcvg\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-88gnn","name":"long-running-qtcvg"}}
{"level":"info","ts":"2026-07-13T15:53:53.421Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4e652317-0e37-4b89-8796-dfcde35e3f93"}
{"level":"error","ts":"2026-07-13T15:53:53.421Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4e652317-0e37-4b89-8796-dfcde35e3f93","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.421Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"44f99fa7-e8bb-4066-a8ed-8d1e753e6729","name":"long-running-qtcvg-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:53:53.421Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9eafb610-d5c1-42bc-8885-73d3bc56c2c5"}
{"level":"info","ts":"2026-07-13T15:53:53.421Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9eafb610-d5c1-42bc-8885-73d3bc56c2c5"}
{"level":"info","ts":"2026-07-13T15:53:53.421Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9eafb610-d5c1-42bc-8885-73d3bc56c2c5","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:53.421Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9eafb610-d5c1-42bc-8885-73d3bc56c2c5","error":"head certificate secret ray-head-secret-long-running-qtcvg not found"}
{"level":"error","ts":"2026-07-13T15:53:53.421Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9eafb610-d5c1-42bc-8885-73d3bc56c2c5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.421Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9eafb610-d5c1-42bc-8885-73d3bc56c2c5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found"}
{"level":"info","ts":"2026-07-13T15:53:53.422Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9eafb610-d5c1-42bc-8885-73d3bc56c2c5","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:53.422Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9eafb610-d5c1-42bc-8885-73d3bc56c2c5","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:53.422Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9eafb610-d5c1-42bc-8885-73d3bc56c2c5","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:53.422Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9eafb610-d5c1-42bc-8885-73d3bc56c2c5","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:53:53.422Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9eafb610-d5c1-42bc-8885-73d3bc56c2c5","name":"long-running-qtcvg","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-qtcvg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:53:53.422Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"97045da1-0194-4a17-a57a-e23c07535bfc","rayCluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.422Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"97045da1-0194-4a17-a57a-e23c07535bfc","rayCluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.429Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"e50eecda-9c69-4ba9-9166-d306aebf3ed1","name":"long-running-qtcvg-head"}
{"level":"info","ts":"2026-07-13T15:53:53.429Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"44f99fa7-e8bb-4066-a8ed-8d1e753e6729","name":"kuberay-gateway-access","namespace":"test-ns-88gnn","operation":"created"}
{"level":"info","ts":"2026-07-13T15:53:53.433Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9eafb610-d5c1-42bc-8885-73d3bc56c2c5"}
{"level":"error","ts":"2026-07-13T15:53:53.521Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9eafb610-d5c1-42bc-8885-73d3bc56c2c5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.433Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"6b4371b9-8acf-49bd-b4e5-9c833fd32d18","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:53:53.521Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"6b4371b9-8acf-49bd-b4e5-9c833fd32d18","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.521Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"6b4371b9-8acf-49bd-b4e5-9c833fd32d18","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:53:53.521Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9ff8fe7b-2383-4964-ae51-5a786a8aee30"}
{"level":"info","ts":"2026-07-13T15:53:53.521Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9ff8fe7b-2383-4964-ae51-5a786a8aee30"}
{"level":"info","ts":"2026-07-13T15:53:53.521Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9ff8fe7b-2383-4964-ae51-5a786a8aee30","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:53.521Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9ff8fe7b-2383-4964-ae51-5a786a8aee30","error":"head certificate secret ray-head-secret-long-running-qtcvg not found"}
{"level":"error","ts":"2026-07-13T15:53:53.521Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9ff8fe7b-2383-4964-ae51-5a786a8aee30","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.521Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9ff8fe7b-2383-4964-ae51-5a786a8aee30","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found"}
{"level":"info","ts":"2026-07-13T15:53:53.521Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9ff8fe7b-2383-4964-ae51-5a786a8aee30","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:53.521Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9ff8fe7b-2383-4964-ae51-5a786a8aee30","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:53.521Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"6b4371b9-8acf-49bd-b4e5-9c833fd32d18"}
{"level":"info","ts":"2026-07-13T15:53:53.521Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"6b4371b9-8acf-49bd-b4e5-9c833fd32d18","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:53:53.521Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9ff8fe7b-2383-4964-ae51-5a786a8aee30","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:53.521Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9ff8fe7b-2383-4964-ae51-5a786a8aee30"}
{"level":"error","ts":"2026-07-13T15:53:53.521Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9ff8fe7b-2383-4964-ae51-5a786a8aee30","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.522Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"44f99fa7-e8bb-4066-a8ed-8d1e753e6729","name":"test-ns-88gnn-long-running-qtcvg","operation":"created"}
{"level":"info","ts":"2026-07-13T15:53:53.522Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bc97aead-1200-4e71-91d1-e1d35fecbfa0"}
{"level":"info","ts":"2026-07-13T15:53:53.522Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bc97aead-1200-4e71-91d1-e1d35fecbfa0"}
{"level":"info","ts":"2026-07-13T15:53:53.522Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bc97aead-1200-4e71-91d1-e1d35fecbfa0","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:53.522Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bc97aead-1200-4e71-91d1-e1d35fecbfa0","error":"head certificate secret ray-head-secret-long-running-qtcvg not found"}
{"level":"error","ts":"2026-07-13T15:53:53.522Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bc97aead-1200-4e71-91d1-e1d35fecbfa0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.522Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bc97aead-1200-4e71-91d1-e1d35fecbfa0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found"}
{"level":"info","ts":"2026-07-13T15:53:53.522Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bc97aead-1200-4e71-91d1-e1d35fecbfa0","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:53.522Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bc97aead-1200-4e71-91d1-e1d35fecbfa0","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:53.522Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bc97aead-1200-4e71-91d1-e1d35fecbfa0","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:53.522Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bc97aead-1200-4e71-91d1-e1d35fecbfa0"}
{"level":"error","ts":"2026-07-13T15:53:53.522Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bc97aead-1200-4e71-91d1-e1d35fecbfa0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.522Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"97045da1-0194-4a17-a57a-e23c07535bfc","rayCluster":"long-running-qtcvg","secret":"ca-secret"}
{"level":"info","ts":"2026-07-13T15:53:53.522Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"97045da1-0194-4a17-a57a-e23c07535bfc","rayCluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.528Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"44f99fa7-e8bb-4066-a8ed-8d1e753e6729","name":"kube-rbac-proxy-config-long-running-qtcvg","operation":"created"}
{"level":"info","ts":"2026-07-13T15:53:53.528Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"e50eecda-9c69-4ba9-9166-d306aebf3ed1","name":"long-running-qtcvg-workers"}
{"level":"info","ts":"2026-07-13T15:53:53.528Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"e50eecda-9c69-4ba9-9166-d306aebf3ed1","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.528Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"92151254-befe-4daa-a253-5fb2da47a06d","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.532Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"97045da1-0194-4a17-a57a-e23c07535bfc","rayCluster":"long-running-qtcvg"}
{"level":"error","ts":"2026-07-13T15:53:53.534Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"44f99fa7-e8bb-4066-a8ed-8d1e753e6729","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qtcvg\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.534Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"44f99fa7-e8bb-4066-a8ed-8d1e753e6729"}
{"level":"error","ts":"2026-07-13T15:53:53.534Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"44f99fa7-e8bb-4066-a8ed-8d1e753e6729","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qtcvg\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.534Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"2de8cac2-225f-41f5-83c8-95126f8aaab8","namespacedName":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:53:53.534Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"2de8cac2-225f-41f5-83c8-95126f8aaab8","mode":"IntegratedOAuth","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.534Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"2de8cac2-225f-41f5-83c8-95126f8aaab8","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.540Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"92151254-befe-4daa-a253-5fb2da47a06d","name":"long-running-qtcvg-head"}
{"level":"info","ts":"2026-07-13T15:53:53.541Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"97045da1-0194-4a17-a57a-e23c07535bfc","rayCluster":"long-running-qtcvg","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:53:53.542Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"57ee2b28-a820-43a1-bb50-0078401da1f9"}
{"level":"info","ts":"2026-07-13T15:53:53.542Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"57ee2b28-a820-43a1-bb50-0078401da1f9"}
{"level":"info","ts":"2026-07-13T15:53:53.542Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"57ee2b28-a820-43a1-bb50-0078401da1f9","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:53.542Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"57ee2b28-a820-43a1-bb50-0078401da1f9","error":"head certificate secret ray-head-secret-long-running-qtcvg not found"}
{"level":"error","ts":"2026-07-13T15:53:53.542Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"57ee2b28-a820-43a1-bb50-0078401da1f9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.542Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"57ee2b28-a820-43a1-bb50-0078401da1f9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found"}
{"level":"info","ts":"2026-07-13T15:53:53.542Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"57ee2b28-a820-43a1-bb50-0078401da1f9","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:53.542Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"57ee2b28-a820-43a1-bb50-0078401da1f9","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:53.542Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"57ee2b28-a820-43a1-bb50-0078401da1f9","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:53.542Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"57ee2b28-a820-43a1-bb50-0078401da1f9"}
{"level":"error","ts":"2026-07-13T15:53:53.542Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"57ee2b28-a820-43a1-bb50-0078401da1f9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.621Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"2de8cac2-225f-41f5-83c8-95126f8aaab8","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.622Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"3a5c30ba-ed38-4dea-8485-34e13cf39d06","namespacedName":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:53:53.622Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"3a5c30ba-ed38-4dea-8485-34e13cf39d06","mode":"IntegratedOAuth","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.622Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"3a5c30ba-ed38-4dea-8485-34e13cf39d06","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.625Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"97045da1-0194-4a17-a57a-e23c07535bfc","rayCluster":"long-running-qtcvg","podIPs":[]}
{"level":"error","ts":"2026-07-13T15:53:53.625Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"97045da1-0194-4a17-a57a-e23c07535bfc","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-qtcvg\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.625Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"97045da1-0194-4a17-a57a-e23c07535bfc"}
{"level":"error","ts":"2026-07-13T15:53:53.626Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"97045da1-0194-4a17-a57a-e23c07535bfc","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-qtcvg\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.626Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"bf4e21b3-db07-485b-82d0-7524a5f23772","rayCluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.626Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b9c4528a-90fa-4103-94be-8edf291e8996","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:53:53.626Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b9c4528a-90fa-4103-94be-8edf291e8996","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.626Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b9c4528a-90fa-4103-94be-8edf291e8996","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:53:53.626Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b9c4528a-90fa-4103-94be-8edf291e8996"}
{"level":"info","ts":"2026-07-13T15:53:53.626Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b9c4528a-90fa-4103-94be-8edf291e8996","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:53:53.627Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872c16e8-02d3-4177-953c-c1f5c4111328"}
{"level":"info","ts":"2026-07-13T15:53:53.627Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872c16e8-02d3-4177-953c-c1f5c4111328"}
{"level":"info","ts":"2026-07-13T15:53:53.627Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872c16e8-02d3-4177-953c-c1f5c4111328","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:53.627Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872c16e8-02d3-4177-953c-c1f5c4111328","error":"head certificate secret ray-head-secret-long-running-qtcvg not found"}
{"level":"error","ts":"2026-07-13T15:53:53.627Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872c16e8-02d3-4177-953c-c1f5c4111328","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.627Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872c16e8-02d3-4177-953c-c1f5c4111328","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found"}
{"level":"info","ts":"2026-07-13T15:53:53.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872c16e8-02d3-4177-953c-c1f5c4111328","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:53.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872c16e8-02d3-4177-953c-c1f5c4111328","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:53.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872c16e8-02d3-4177-953c-c1f5c4111328","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:53.627Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872c16e8-02d3-4177-953c-c1f5c4111328"}
{"level":"error","ts":"2026-07-13T15:53:53.628Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872c16e8-02d3-4177-953c-c1f5c4111328","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.631Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"92151254-befe-4daa-a253-5fb2da47a06d","name":"long-running-qtcvg-workers"}
{"level":"info","ts":"2026-07-13T15:53:53.631Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"92151254-befe-4daa-a253-5fb2da47a06d","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.632Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"d26d2a82-36c4-4aef-ba3f-d852c894e3af","cluster":"long-running-qtcvg"}
{"level":"error","ts":"2026-07-13T15:53:53.632Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"3a5c30ba-ed38-4dea-8485-34e13cf39d06","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qtcvg\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.632Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"3a5c30ba-ed38-4dea-8485-34e13cf39d06"}
{"level":"error","ts":"2026-07-13T15:53:53.632Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"3a5c30ba-ed38-4dea-8485-34e13cf39d06","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qtcvg\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.637Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"bf30e774-b90b-4791-ad31-2c1217009ed8","namespacedName":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:53:53.637Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"bf30e774-b90b-4791-ad31-2c1217009ed8","mode":"IntegratedOAuth","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.637Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"bf30e774-b90b-4791-ad31-2c1217009ed8","cluster":"long-running-qtcvg"}
{"level":"error","ts":"2026-07-13T15:53:53.638Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"bf4e21b3-db07-485b-82d0-7524a5f23772","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-qtcvg\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:53:53.638Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"bf4e21b3-db07-485b-82d0-7524a5f23772","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-qtcvg\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.638Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"bf4e21b3-db07-485b-82d0-7524a5f23772"}
{"level":"error","ts":"2026-07-13T15:53:53.638Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"bf4e21b3-db07-485b-82d0-7524a5f23772","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-qtcvg\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.638Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"4a2cc17a-b7b7-47d5-955f-7ff6adc69d86","rayCluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.640Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"4a2cc17a-b7b7-47d5-955f-7ff6adc69d86"}
{"level":"info","ts":"2026-07-13T15:53:53.646Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"d26d2a82-36c4-4aef-ba3f-d852c894e3af","name":"long-running-qtcvg-head"}
{"level":"info","ts":"2026-07-13T15:53:53.721Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"428ebdb7-cb15-443a-a0e6-67712adc37c6","rayCluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.721Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"428ebdb7-cb15-443a-a0e6-67712adc37c6"}
{"level":"info","ts":"2026-07-13T15:53:53.722Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"bf30e774-b90b-4791-ad31-2c1217009ed8","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.726Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"d26d2a82-36c4-4aef-ba3f-d852c894e3af","name":"long-running-qtcvg-workers"}
{"level":"info","ts":"2026-07-13T15:53:53.726Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"d26d2a82-36c4-4aef-ba3f-d852c894e3af","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:53.788Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8190cbd7-c53d-4d4b-8f79-74542286bd38"}
{"level":"info","ts":"2026-07-13T15:53:53.789Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8190cbd7-c53d-4d4b-8f79-74542286bd38"}
{"level":"info","ts":"2026-07-13T15:53:53.789Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8190cbd7-c53d-4d4b-8f79-74542286bd38","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:53.789Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8190cbd7-c53d-4d4b-8f79-74542286bd38","error":"head certificate secret ray-head-secret-long-running-qtcvg not found"}
{"level":"error","ts":"2026-07-13T15:53:53.789Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8190cbd7-c53d-4d4b-8f79-74542286bd38","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:53.789Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8190cbd7-c53d-4d4b-8f79-74542286bd38","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found"}
{"level":"info","ts":"2026-07-13T15:53:53.789Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8190cbd7-c53d-4d4b-8f79-74542286bd38","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:53.789Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8190cbd7-c53d-4d4b-8f79-74542286bd38","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:53.789Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8190cbd7-c53d-4d4b-8f79-74542286bd38","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:53.789Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8190cbd7-c53d-4d4b-8f79-74542286bd38"}
{"level":"error","ts":"2026-07-13T15:53:53.789Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8190cbd7-c53d-4d4b-8f79-74542286bd38","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:54.109Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"b8cedb5f-0b44-49bd-ba93-16f12c64f1f0"}
{"level":"info","ts":"2026-07-13T15:53:54.109Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"b8cedb5f-0b44-49bd-ba93-16f12c64f1f0"}
{"level":"info","ts":"2026-07-13T15:53:54.109Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"b8cedb5f-0b44-49bd-ba93-16f12c64f1f0","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:54.109Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"b8cedb5f-0b44-49bd-ba93-16f12c64f1f0","error":"head certificate secret ray-head-secret-long-running-qtcvg not found"}
{"level":"error","ts":"2026-07-13T15:53:54.109Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"b8cedb5f-0b44-49bd-ba93-16f12c64f1f0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:54.109Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"b8cedb5f-0b44-49bd-ba93-16f12c64f1f0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found"}
{"level":"info","ts":"2026-07-13T15:53:54.109Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"b8cedb5f-0b44-49bd-ba93-16f12c64f1f0","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:54.110Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"b8cedb5f-0b44-49bd-ba93-16f12c64f1f0","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:54.110Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"b8cedb5f-0b44-49bd-ba93-16f12c64f1f0","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:54.110Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"b8cedb5f-0b44-49bd-ba93-16f12c64f1f0"}
{"level":"error","ts":"2026-07-13T15:53:54.110Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"b8cedb5f-0b44-49bd-ba93-16f12c64f1f0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:54.751Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8bcf47ec-5f90-4d11-a554-4da875374ecf"}
{"level":"info","ts":"2026-07-13T15:53:54.751Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8bcf47ec-5f90-4d11-a554-4da875374ecf"}
{"level":"info","ts":"2026-07-13T15:53:54.751Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8bcf47ec-5f90-4d11-a554-4da875374ecf","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:54.751Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8bcf47ec-5f90-4d11-a554-4da875374ecf","error":"head certificate secret ray-head-secret-long-running-qtcvg not found"}
{"level":"error","ts":"2026-07-13T15:53:54.751Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8bcf47ec-5f90-4d11-a554-4da875374ecf","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:54.751Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8bcf47ec-5f90-4d11-a554-4da875374ecf","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found"}
{"level":"info","ts":"2026-07-13T15:53:54.751Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8bcf47ec-5f90-4d11-a554-4da875374ecf","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:54.751Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8bcf47ec-5f90-4d11-a554-4da875374ecf","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:54.751Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8bcf47ec-5f90-4d11-a554-4da875374ecf","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qtcvg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:54.751Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8bcf47ec-5f90-4d11-a554-4da875374ecf"}
{"level":"error","ts":"2026-07-13T15:53:54.751Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"8bcf47ec-5f90-4d11-a554-4da875374ecf","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qtcvg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:55.916Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-5xvz7"},"reconcileID":"e13ad809-4aec-4be0-9fd3-f60f39940368"}
{"level":"info","ts":"2026-07-13T15:53:56.031Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86"}
{"level":"info","ts":"2026-07-13T15:53:56.031Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86"}
{"level":"info","ts":"2026-07-13T15:53:56.031Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:56.032Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86"}
{"level":"info","ts":"2026-07-13T15:53:56.032Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:53:56.032Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86"}
{"level":"info","ts":"2026-07-13T15:53:56.032Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","mode":"IntegratedOAuth","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:56.032Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","cluster":"long-running-qtcvg","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:53:56.032Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","cluster":"long-running-qtcvg","authType":"IntegratedOAuth","serviceAccount":"long-running-qtcvg-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T15:53:56.032Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-qtcvg","ray.io/group":"headgroup","ray.io/identifier":"long-running-qtcvg-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T15:53:56.032Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-qtcvg-head-svc.test-ns-88gnn.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T15:53:56.032Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-qtcvg-head-svc.test-ns-88gnn.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:53:56.032Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-qtcvg-head-svc.test-ns-88gnn.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:53:56.032Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","enabled":true}
{"level":"info","ts":"2026-07-13T15:53:56.051Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","name":"long-running-qtcvg-head-44cnb"}
{"level":"info","ts":"2026-07-13T15:53:56.051Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:56.051Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:56.051Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T15:53:56.051Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:56.051Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T15:53:56.051Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86"}
{"level":"info","ts":"2026-07-13T15:53:56.051Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","nodeType":"worker","rayStartParams":{"address":"long-running-qtcvg-head-svc.test-ns-88gnn.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T15:53:56.051Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","rayStartCmd":"ray start  --address=long-running-qtcvg-head-svc.test-ns-88gnn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:53:56.051Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-qtcvg-head-svc.test-ns-88gnn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:53:56.051Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","enabled":true}
{"level":"info","ts":"2026-07-13T15:53:56.064Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","name":"long-running-qtcvg-small-group-worker-4jbht"}
{"level":"info","ts":"2026-07-13T15:53:56.065Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","podName":"long-running-qtcvg-head-44cnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:56.065Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","podName":"long-running-qtcvg-head-44cnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:56.065Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-qtcvg-head-svc"},"newHead":{"podName":"long-running-qtcvg-head-44cnb","serviceName":"long-running-qtcvg-head-svc"}}
{"level":"info","ts":"2026-07-13T15:53:56.065Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"7013986d-c22c-4ebb-a6aa-8c38968b2c86","name":"long-running-qtcvg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-qtcvg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qtcvg-head-44cnb","serviceName":"long-running-qtcvg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:53:56.223Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"6668d577-d022-42a7-a49e-dce4538f0230"}
{"level":"info","ts":"2026-07-13T15:53:56.223Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"6668d577-d022-42a7-a49e-dce4538f0230"}
{"level":"info","ts":"2026-07-13T15:53:56.223Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"6668d577-d022-42a7-a49e-dce4538f0230","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:56.223Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"6668d577-d022-42a7-a49e-dce4538f0230","Found 1 head Pod":"long-running-qtcvg-head-44cnb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:56.223Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"6668d577-d022-42a7-a49e-dce4538f0230","head Pod":"long-running-qtcvg-head-44cnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qtcvg-head-44cnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:56.223Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"0e2ff8fd-756f-4625-bad1-9d341e0f9ff9","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:56.223Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"6668d577-d022-42a7-a49e-dce4538f0230","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:56.223Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"87460a6e-bd8a-41d7-aaa2-a4eb16c05b0b","rayCluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:56.223Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"6668d577-d022-42a7-a49e-dce4538f0230","worker Pod":"long-running-qtcvg-small-group-worker-4jbht","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qtcvg-small-group-worker-4jbht. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:56.223Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"6668d577-d022-42a7-a49e-dce4538f0230","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:56.223Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"6668d577-d022-42a7-a49e-dce4538f0230","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:53:56.223Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"6668d577-d022-42a7-a49e-dce4538f0230","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:56.223Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"84bd8206-1cd5-449a-a39a-cd6509c96c19","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:53:56.223Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"84bd8206-1cd5-449a-a39a-cd6509c96c19","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:56.223Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"84bd8206-1cd5-449a-a39a-cd6509c96c19","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:53:56.223Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"87460a6e-bd8a-41d7-aaa2-a4eb16c05b0b","rayCluster":"long-running-qtcvg","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:53:56.223Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"84bd8206-1cd5-449a-a39a-cd6509c96c19"}
{"level":"info","ts":"2026-07-13T15:53:56.223Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"84bd8206-1cd5-449a-a39a-cd6509c96c19","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:53:56.321Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"6668d577-d022-42a7-a49e-dce4538f0230","podName":"long-running-qtcvg-head-44cnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:56.321Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"6668d577-d022-42a7-a49e-dce4538f0230","podName":"long-running-qtcvg-head-44cnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:56.321Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"bb0079a6-d9e4-4af9-a853-199e4efc9f85","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:53:56.321Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"6668d577-d022-42a7-a49e-dce4538f0230","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-qtcvg-head-svc"},"newHead":{"podName":"long-running-qtcvg-head-44cnb","serviceName":"long-running-qtcvg-head-svc"}}
{"level":"info","ts":"2026-07-13T15:53:56.321Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"6668d577-d022-42a7-a49e-dce4538f0230","name":"long-running-qtcvg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-qtcvg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qtcvg-head-44cnb","serviceName":"long-running-qtcvg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:53:56.321Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"bb0079a6-d9e4-4af9-a853-199e4efc9f85","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:56.321Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"bb0079a6-d9e4-4af9-a853-199e4efc9f85","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:53:56.321Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"bb0079a6-d9e4-4af9-a853-199e4efc9f85"}
{"level":"info","ts":"2026-07-13T15:53:56.321Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"bb0079a6-d9e4-4af9-a853-199e4efc9f85","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:53:56.325Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"0e2ff8fd-756f-4625-bad1-9d341e0f9ff9","name":"long-running-qtcvg-head"}
{"level":"info","ts":"2026-07-13T15:53:56.327Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"6668d577-d022-42a7-a49e-dce4538f0230","name":"long-running-qtcvg","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qtcvg\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-88gnn","name":"long-running-qtcvg"}}
{"level":"info","ts":"2026-07-13T15:53:56.327Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"6668d577-d022-42a7-a49e-dce4538f0230"}
{"level":"error","ts":"2026-07-13T15:53:56.327Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"6668d577-d022-42a7-a49e-dce4538f0230","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qtcvg\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:56.327Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"ca00d69f-9cc3-4cbf-8877-7155e522ba55"}
{"level":"info","ts":"2026-07-13T15:53:56.327Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"ca00d69f-9cc3-4cbf-8877-7155e522ba55"}
{"level":"info","ts":"2026-07-13T15:53:56.327Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"ca00d69f-9cc3-4cbf-8877-7155e522ba55","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:56.327Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"ca00d69f-9cc3-4cbf-8877-7155e522ba55","Found 1 head Pod":"long-running-qtcvg-head-44cnb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:56.327Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"ca00d69f-9cc3-4cbf-8877-7155e522ba55","head Pod":"long-running-qtcvg-head-44cnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qtcvg-head-44cnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:56.327Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"ca00d69f-9cc3-4cbf-8877-7155e522ba55","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:56.327Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"ca00d69f-9cc3-4cbf-8877-7155e522ba55","worker Pod":"long-running-qtcvg-small-group-worker-4jbht","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qtcvg-small-group-worker-4jbht. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:56.327Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"ca00d69f-9cc3-4cbf-8877-7155e522ba55","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:56.327Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"ca00d69f-9cc3-4cbf-8877-7155e522ba55","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:53:56.327Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"ca00d69f-9cc3-4cbf-8877-7155e522ba55","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:56.327Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"ca00d69f-9cc3-4cbf-8877-7155e522ba55","podName":"long-running-qtcvg-head-44cnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:56.328Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"ca00d69f-9cc3-4cbf-8877-7155e522ba55","podName":"long-running-qtcvg-head-44cnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:56.328Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"ca00d69f-9cc3-4cbf-8877-7155e522ba55","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:53:56.328Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"ca00d69f-9cc3-4cbf-8877-7155e522ba55","name":"long-running-qtcvg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qtcvg-head-44cnb","serviceName":"long-running-qtcvg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qtcvg-head-44cnb","serviceName":"long-running-qtcvg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:53:56.337Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"0e2ff8fd-756f-4625-bad1-9d341e0f9ff9","name":"long-running-qtcvg-workers"}
{"level":"info","ts":"2026-07-13T15:53:56.337Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"0e2ff8fd-756f-4625-bad1-9d341e0f9ff9","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:56.338Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"062a0008-56ee-4f85-bb99-c3a170427937"}
{"level":"info","ts":"2026-07-13T15:53:56.338Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"062a0008-56ee-4f85-bb99-c3a170427937"}
{"level":"info","ts":"2026-07-13T15:53:56.338Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"062a0008-56ee-4f85-bb99-c3a170427937","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:56.338Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"062a0008-56ee-4f85-bb99-c3a170427937","Found 1 head Pod":"long-running-qtcvg-head-44cnb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:56.338Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"062a0008-56ee-4f85-bb99-c3a170427937","head Pod":"long-running-qtcvg-head-44cnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qtcvg-head-44cnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:56.338Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"062a0008-56ee-4f85-bb99-c3a170427937","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:56.338Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"062a0008-56ee-4f85-bb99-c3a170427937","worker Pod":"long-running-qtcvg-small-group-worker-4jbht","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qtcvg-small-group-worker-4jbht. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:56.338Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"062a0008-56ee-4f85-bb99-c3a170427937","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:56.338Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"062a0008-56ee-4f85-bb99-c3a170427937","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:53:56.338Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"062a0008-56ee-4f85-bb99-c3a170427937","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:56.338Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"e606b867-0324-4481-8d61-78d9329ee42a","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:56.338Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"062a0008-56ee-4f85-bb99-c3a170427937","podName":"long-running-qtcvg-head-44cnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:56.338Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"062a0008-56ee-4f85-bb99-c3a170427937","podName":"long-running-qtcvg-head-44cnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:56.338Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"13e449bb-00a3-472d-99d8-777cc2042228","rayCluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:56.338Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"5209338b-5663-43da-832b-8b411a0a6620","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:53:56.338Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"5209338b-5663-43da-832b-8b411a0a6620","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:56.338Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"062a0008-56ee-4f85-bb99-c3a170427937","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:53:56.339Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"5209338b-5663-43da-832b-8b411a0a6620","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:53:56.339Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"062a0008-56ee-4f85-bb99-c3a170427937","name":"long-running-qtcvg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qtcvg-head-44cnb","serviceName":"long-running-qtcvg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qtcvg-head-44cnb","serviceName":"long-running-qtcvg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:53:56.339Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"13e449bb-00a3-472d-99d8-777cc2042228","rayCluster":"long-running-qtcvg","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:53:56.339Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"5209338b-5663-43da-832b-8b411a0a6620"}
{"level":"info","ts":"2026-07-13T15:53:56.339Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"5209338b-5663-43da-832b-8b411a0a6620","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:53:56.345Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"062a0008-56ee-4f85-bb99-c3a170427937","name":"long-running-qtcvg","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qtcvg\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-88gnn","name":"long-running-qtcvg"}}
{"level":"info","ts":"2026-07-13T15:53:56.345Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"062a0008-56ee-4f85-bb99-c3a170427937"}
{"level":"error","ts":"2026-07-13T15:53:56.345Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"062a0008-56ee-4f85-bb99-c3a170427937","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qtcvg\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:56.421Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0eb56655-413d-45b2-b743-e239d8d2f707"}
{"level":"info","ts":"2026-07-13T15:53:56.421Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0eb56655-413d-45b2-b743-e239d8d2f707"}
{"level":"info","ts":"2026-07-13T15:53:56.421Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0eb56655-413d-45b2-b743-e239d8d2f707","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:56.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0eb56655-413d-45b2-b743-e239d8d2f707","Found 1 head Pod":"long-running-qtcvg-head-44cnb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:56.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0eb56655-413d-45b2-b743-e239d8d2f707","head Pod":"long-running-qtcvg-head-44cnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qtcvg-head-44cnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:56.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0eb56655-413d-45b2-b743-e239d8d2f707","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:56.422Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0eb56655-413d-45b2-b743-e239d8d2f707","worker Pod":"long-running-qtcvg-small-group-worker-4jbht","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qtcvg-small-group-worker-4jbht. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:56.422Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0eb56655-413d-45b2-b743-e239d8d2f707","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:56.422Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0eb56655-413d-45b2-b743-e239d8d2f707","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:53:56.422Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0eb56655-413d-45b2-b743-e239d8d2f707","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:56.422Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0eb56655-413d-45b2-b743-e239d8d2f707","podName":"long-running-qtcvg-head-44cnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:56.422Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0eb56655-413d-45b2-b743-e239d8d2f707","podName":"long-running-qtcvg-head-44cnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:56.422Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0eb56655-413d-45b2-b743-e239d8d2f707","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:53:56.422Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0eb56655-413d-45b2-b743-e239d8d2f707","seconds":300}
{"level":"info","ts":"2026-07-13T15:53:56.422Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"e606b867-0324-4481-8d61-78d9329ee42a","name":"long-running-qtcvg-head"}
{"level":"info","ts":"2026-07-13T15:53:56.433Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"e606b867-0324-4481-8d61-78d9329ee42a","name":"long-running-qtcvg-workers"}
{"level":"info","ts":"2026-07-13T15:53:56.433Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"e606b867-0324-4481-8d61-78d9329ee42a","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:56.498Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2310cccb-f2c4-4f2f-83a4-6ffbc96518a5"}
{"level":"info","ts":"2026-07-13T15:53:56.498Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2310cccb-f2c4-4f2f-83a4-6ffbc96518a5"}
{"level":"info","ts":"2026-07-13T15:53:56.498Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2310cccb-f2c4-4f2f-83a4-6ffbc96518a5","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:56.498Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2310cccb-f2c4-4f2f-83a4-6ffbc96518a5","Found 1 head Pod":"long-running-qtcvg-head-44cnb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:56.498Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2310cccb-f2c4-4f2f-83a4-6ffbc96518a5","head Pod":"long-running-qtcvg-head-44cnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qtcvg-head-44cnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:56.498Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2310cccb-f2c4-4f2f-83a4-6ffbc96518a5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:56.498Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2310cccb-f2c4-4f2f-83a4-6ffbc96518a5","worker Pod":"long-running-qtcvg-small-group-worker-4jbht","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qtcvg-small-group-worker-4jbht. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:56.498Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2310cccb-f2c4-4f2f-83a4-6ffbc96518a5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:56.498Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2310cccb-f2c4-4f2f-83a4-6ffbc96518a5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:53:56.498Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2310cccb-f2c4-4f2f-83a4-6ffbc96518a5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:56.498Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2310cccb-f2c4-4f2f-83a4-6ffbc96518a5","podName":"long-running-qtcvg-head-44cnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:56.498Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2310cccb-f2c4-4f2f-83a4-6ffbc96518a5","podName":"long-running-qtcvg-head-44cnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:56.498Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2310cccb-f2c4-4f2f-83a4-6ffbc96518a5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:53:56.498Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2310cccb-f2c4-4f2f-83a4-6ffbc96518a5","seconds":300}
{"level":"info","ts":"2026-07-13T15:53:56.516Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2e5c4dc5-ad44-4550-8c4b-bc6e6ce9756c"}
{"level":"info","ts":"2026-07-13T15:53:56.516Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2e5c4dc5-ad44-4550-8c4b-bc6e6ce9756c"}
{"level":"info","ts":"2026-07-13T15:53:56.516Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2e5c4dc5-ad44-4550-8c4b-bc6e6ce9756c","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:56.516Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2e5c4dc5-ad44-4550-8c4b-bc6e6ce9756c","Found 1 head Pod":"long-running-qtcvg-head-44cnb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:56.516Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2e5c4dc5-ad44-4550-8c4b-bc6e6ce9756c","head Pod":"long-running-qtcvg-head-44cnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qtcvg-head-44cnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:56.516Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2e5c4dc5-ad44-4550-8c4b-bc6e6ce9756c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:56.516Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2e5c4dc5-ad44-4550-8c4b-bc6e6ce9756c","worker Pod":"long-running-qtcvg-small-group-worker-4jbht","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qtcvg-small-group-worker-4jbht. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:56.516Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2e5c4dc5-ad44-4550-8c4b-bc6e6ce9756c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:56.516Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2e5c4dc5-ad44-4550-8c4b-bc6e6ce9756c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:53:56.516Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2e5c4dc5-ad44-4550-8c4b-bc6e6ce9756c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:56.516Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2e5c4dc5-ad44-4550-8c4b-bc6e6ce9756c","podName":"long-running-qtcvg-small-group-worker-4jbht","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:56.516Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2e5c4dc5-ad44-4550-8c4b-bc6e6ce9756c","podName":"long-running-qtcvg-small-group-worker-4jbht","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:56.516Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2e5c4dc5-ad44-4550-8c4b-bc6e6ce9756c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:53:56.516Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2e5c4dc5-ad44-4550-8c4b-bc6e6ce9756c","seconds":300}
{"level":"info","ts":"2026-07-13T15:53:56.795Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0da2601d-76b4-47df-9cbe-3086fda42ce8"}
{"level":"info","ts":"2026-07-13T15:53:56.795Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0da2601d-76b4-47df-9cbe-3086fda42ce8"}
{"level":"info","ts":"2026-07-13T15:53:56.795Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0da2601d-76b4-47df-9cbe-3086fda42ce8","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:56.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0da2601d-76b4-47df-9cbe-3086fda42ce8","Found 1 head Pod":"long-running-qtcvg-head-44cnb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:56.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0da2601d-76b4-47df-9cbe-3086fda42ce8","head Pod":"long-running-qtcvg-head-44cnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qtcvg-head-44cnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:56.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0da2601d-76b4-47df-9cbe-3086fda42ce8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:56.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0da2601d-76b4-47df-9cbe-3086fda42ce8","worker Pod":"long-running-qtcvg-small-group-worker-4jbht","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qtcvg-small-group-worker-4jbht. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:56.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0da2601d-76b4-47df-9cbe-3086fda42ce8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:56.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0da2601d-76b4-47df-9cbe-3086fda42ce8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:53:56.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0da2601d-76b4-47df-9cbe-3086fda42ce8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:56.795Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0da2601d-76b4-47df-9cbe-3086fda42ce8","podName":"long-running-qtcvg-head-44cnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:56.795Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0da2601d-76b4-47df-9cbe-3086fda42ce8","podName":"long-running-qtcvg-head-44cnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:56.795Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0da2601d-76b4-47df-9cbe-3086fda42ce8","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"long-running-qtcvg-head-44cnb","serviceName":"long-running-qtcvg-head-svc"},"newHead":{"podIP":"10.133.0.22","serviceIP":"10.133.0.22","podName":"long-running-qtcvg-head-44cnb","serviceName":"long-running-qtcvg-head-svc"}}
{"level":"info","ts":"2026-07-13T15:53:56.795Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0da2601d-76b4-47df-9cbe-3086fda42ce8","name":"long-running-qtcvg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qtcvg-head-44cnb","serviceName":"long-running-qtcvg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.22","serviceIP":"10.133.0.22","podName":"long-running-qtcvg-head-44cnb","serviceName":"long-running-qtcvg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:53:56.807Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"c26f5eec-020a-4de2-ac95-30e83fee7af9","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:56.807Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"eb03df21-79a3-4828-8342-35384da4d989","rayCluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:56.807Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"370a0e4b-8257-4621-8bf4-db1c30e5a26d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:53:56.807Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"370a0e4b-8257-4621-8bf4-db1c30e5a26d","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:56.807Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"370a0e4b-8257-4621-8bf4-db1c30e5a26d","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:53:56.807Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"eb03df21-79a3-4828-8342-35384da4d989","pod":"long-running-qtcvg-head-44cnb","ip":"10.133.0.22"}
{"level":"info","ts":"2026-07-13T15:53:56.807Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"370a0e4b-8257-4621-8bf4-db1c30e5a26d"}
{"level":"info","ts":"2026-07-13T15:53:56.807Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"370a0e4b-8257-4621-8bf4-db1c30e5a26d","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:53:56.818Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"c26f5eec-020a-4de2-ac95-30e83fee7af9","name":"long-running-qtcvg-head"}
{"level":"info","ts":"2026-07-13T15:53:56.818Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"eb03df21-79a3-4828-8342-35384da4d989","certificate":"ray-head-cert-long-running-qtcvg","dnsNames":["long-running-qtcvg-head-svc","localhost","long-running-qtcvg-head-svc.test-ns-88gnn.svc","long-running-qtcvg-head-svc.test-ns-88gnn.svc.cluster.local"],"ipAddresses":["10.133.0.22","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:53:56.826Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"c26f5eec-020a-4de2-ac95-30e83fee7af9","name":"long-running-qtcvg-workers"}
{"level":"info","ts":"2026-07-13T15:53:56.826Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"c26f5eec-020a-4de2-ac95-30e83fee7af9","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:56.830Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"eb03df21-79a3-4828-8342-35384da4d989","certificate":"ray-worker-cert-long-running-qtcvg","dnsNames":["long-running-qtcvg-worker-svc","localhost","long-running-qtcvg-small-group","long-running-qtcvg-worker-svc.test-ns-88gnn.svc","long-running-qtcvg-worker-svc.test-ns-88gnn.svc.cluster.local","*.long-running-qtcvg-worker-svc.test-ns-88gnn.svc","*.long-running-qtcvg-worker-svc.test-ns-88gnn.svc.cluster.local","*-worker-*.test-ns-88gnn.svc","*-worker-*.test-ns-88gnn.svc.cluster.local"],"ipAddresses":["10.133.0.22","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:53:56.830Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"eb03df21-79a3-4828-8342-35384da4d989","rayCluster":"long-running-qtcvg","podIPs":["10.133.0.22"]}
{"level":"info","ts":"2026-07-13T15:53:56.954Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"02dcbc58-2c76-4b5e-b337-8f04f5f75ce0"}
{"level":"info","ts":"2026-07-13T15:53:56.954Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"02dcbc58-2c76-4b5e-b337-8f04f5f75ce0"}
{"level":"info","ts":"2026-07-13T15:53:56.954Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"02dcbc58-2c76-4b5e-b337-8f04f5f75ce0","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:56.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"02dcbc58-2c76-4b5e-b337-8f04f5f75ce0","Found 1 head Pod":"long-running-qtcvg-head-44cnb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:56.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"02dcbc58-2c76-4b5e-b337-8f04f5f75ce0","head Pod":"long-running-qtcvg-head-44cnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qtcvg-head-44cnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:56.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"02dcbc58-2c76-4b5e-b337-8f04f5f75ce0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:56.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"02dcbc58-2c76-4b5e-b337-8f04f5f75ce0","worker Pod":"long-running-qtcvg-small-group-worker-4jbht","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qtcvg-small-group-worker-4jbht. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:56.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"02dcbc58-2c76-4b5e-b337-8f04f5f75ce0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:56.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"02dcbc58-2c76-4b5e-b337-8f04f5f75ce0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:53:56.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"02dcbc58-2c76-4b5e-b337-8f04f5f75ce0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:56.954Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"02dcbc58-2c76-4b5e-b337-8f04f5f75ce0","podName":"long-running-qtcvg-head-44cnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:56.954Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"02dcbc58-2c76-4b5e-b337-8f04f5f75ce0","podName":"long-running-qtcvg-head-44cnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:56.954Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"02dcbc58-2c76-4b5e-b337-8f04f5f75ce0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:53:56.954Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"02dcbc58-2c76-4b5e-b337-8f04f5f75ce0","seconds":300}
{"level":"info","ts":"2026-07-13T15:53:58.209Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"0c73931c-9a21-4c0e-a02f-f9a4d3043714"}
{"level":"info","ts":"2026-07-13T15:53:58.209Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"0c73931c-9a21-4c0e-a02f-f9a4d3043714"}
{"level":"info","ts":"2026-07-13T15:53:58.210Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"0c73931c-9a21-4c0e-a02f-f9a4d3043714","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:58.210Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"0c73931c-9a21-4c0e-a02f-f9a4d3043714","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:58.210Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"0c73931c-9a21-4c0e-a02f-f9a4d3043714","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:58.210Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"0c73931c-9a21-4c0e-a02f-f9a4d3043714","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:58.210Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"0c73931c-9a21-4c0e-a02f-f9a4d3043714","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:58.210Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"0c73931c-9a21-4c0e-a02f-f9a4d3043714","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:58.210Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"0c73931c-9a21-4c0e-a02f-f9a4d3043714","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:53:58.210Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"0c73931c-9a21-4c0e-a02f-f9a4d3043714","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:58.210Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"0c73931c-9a21-4c0e-a02f-f9a4d3043714","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:53:58.210Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"0c73931c-9a21-4c0e-a02f-f9a4d3043714","seconds":300}
{"level":"info","ts":"2026-07-13T15:53:58.214Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7a288e00-da56-4d90-8478-320fad1a2b8c"}
{"level":"info","ts":"2026-07-13T15:53:58.214Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7a288e00-da56-4d90-8478-320fad1a2b8c"}
{"level":"info","ts":"2026-07-13T15:53:58.214Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7a288e00-da56-4d90-8478-320fad1a2b8c","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:58.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7a288e00-da56-4d90-8478-320fad1a2b8c","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:58.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7a288e00-da56-4d90-8478-320fad1a2b8c","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:58.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7a288e00-da56-4d90-8478-320fad1a2b8c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:58.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7a288e00-da56-4d90-8478-320fad1a2b8c","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:58.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7a288e00-da56-4d90-8478-320fad1a2b8c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:58.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7a288e00-da56-4d90-8478-320fad1a2b8c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:53:58.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7a288e00-da56-4d90-8478-320fad1a2b8c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:58.215Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7a288e00-da56-4d90-8478-320fad1a2b8c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:53:58.215Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"7a288e00-da56-4d90-8478-320fad1a2b8c","seconds":300}
{"level":"info","ts":"2026-07-13T15:53:58.234Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"470eb9c9-eb74-48f8-a8f6-ca298e942e9c"}
{"level":"info","ts":"2026-07-13T15:53:58.234Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"470eb9c9-eb74-48f8-a8f6-ca298e942e9c"}
{"level":"info","ts":"2026-07-13T15:53:58.235Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"470eb9c9-eb74-48f8-a8f6-ca298e942e9c","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:58.235Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"470eb9c9-eb74-48f8-a8f6-ca298e942e9c","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:58.235Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"470eb9c9-eb74-48f8-a8f6-ca298e942e9c","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:58.235Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"470eb9c9-eb74-48f8-a8f6-ca298e942e9c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:58.235Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"470eb9c9-eb74-48f8-a8f6-ca298e942e9c","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:58.235Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"470eb9c9-eb74-48f8-a8f6-ca298e942e9c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:58.235Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"470eb9c9-eb74-48f8-a8f6-ca298e942e9c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:53:58.235Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"470eb9c9-eb74-48f8-a8f6-ca298e942e9c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:58.235Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"470eb9c9-eb74-48f8-a8f6-ca298e942e9c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:53:58.235Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"470eb9c9-eb74-48f8-a8f6-ca298e942e9c","seconds":300}
{"level":"info","ts":"2026-07-13T15:53:58.237Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"b0a82a70-4b27-47c7-b01e-0d581b7c7bcb"}
{"level":"info","ts":"2026-07-13T15:53:58.237Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"b0a82a70-4b27-47c7-b01e-0d581b7c7bcb"}
{"level":"info","ts":"2026-07-13T15:53:58.237Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"b0a82a70-4b27-47c7-b01e-0d581b7c7bcb","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:58.237Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"b0a82a70-4b27-47c7-b01e-0d581b7c7bcb","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:58.237Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"b0a82a70-4b27-47c7-b01e-0d581b7c7bcb","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:58.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"b0a82a70-4b27-47c7-b01e-0d581b7c7bcb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:58.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"b0a82a70-4b27-47c7-b01e-0d581b7c7bcb","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-pbfxx-small-group-worker-bhr2g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:58.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"b0a82a70-4b27-47c7-b01e-0d581b7c7bcb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:58.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"b0a82a70-4b27-47c7-b01e-0d581b7c7bcb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:53:58.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"b0a82a70-4b27-47c7-b01e-0d581b7c7bcb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:58.238Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"b0a82a70-4b27-47c7-b01e-0d581b7c7bcb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:53:58.238Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"b0a82a70-4b27-47c7-b01e-0d581b7c7bcb","seconds":300}
{"level":"info","ts":"2026-07-13T15:53:58.779Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3676af83-1904-4792-8eac-eac9b860a5f9"}
{"level":"info","ts":"2026-07-13T15:53:58.779Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3676af83-1904-4792-8eac-eac9b860a5f9"}
{"level":"info","ts":"2026-07-13T15:53:58.779Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3676af83-1904-4792-8eac-eac9b860a5f9","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:58.779Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3676af83-1904-4792-8eac-eac9b860a5f9","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:58.779Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3676af83-1904-4792-8eac-eac9b860a5f9","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:58.779Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3676af83-1904-4792-8eac-eac9b860a5f9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:58.779Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3676af83-1904-4792-8eac-eac9b860a5f9","worker Pod":"fail-pbfxx-small-group-worker-bhr2g","shouldDelete":true,"reason":"The worker Pod fail-pbfxx-small-group-worker-bhr2g status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-07-13T15:53:58.782Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3676af83-1904-4792-8eac-eac9b860a5f9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"delete 1 unhealthy worker Pods","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:58.782Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3676af83-1904-4792-8eac-eac9b860a5f9","error":"delete 1 unhealthy worker Pods"}
{"level":"info","ts":"2026-07-13T15:53:58.783Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3676af83-1904-4792-8eac-eac9b860a5f9","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:53:58.783Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3676af83-1904-4792-8eac-eac9b860a5f9","name":"fail-pbfxx","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:51:36Z","stateTransitionTimes":{"ready":"2026-07-13T15:51:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:50:24Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:58Z","stateTransitionTimes":{"ready":"2026-07-13T15:51:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:50:24Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:53:58.794Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3676af83-1904-4792-8eac-eac9b860a5f9"}
{"level":"error","ts":"2026-07-13T15:53:58.794Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"3676af83-1904-4792-8eac-eac9b860a5f9","error":"delete 1 unhealthy worker Pods","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:58.794Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"685ceccd-3c58-40c0-87ff-5ec0cd40020b","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:53:58.794Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"979270d3-af43-4cf5-84a3-b9b6316b31c2","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:53:58.794Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"de013e87-e4ba-4edd-8f39-83129c3dbbeb","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:58.794Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"685ceccd-3c58-40c0-87ff-5ec0cd40020b","pod":"fail-pbfxx-head-27cb9","ip":"10.133.0.20"}
{"level":"info","ts":"2026-07-13T15:53:58.794Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"685ceccd-3c58-40c0-87ff-5ec0cd40020b","pod":"fail-pbfxx-small-group-worker-bhr2g","ip":"10.134.0.26"}
{"level":"info","ts":"2026-07-13T15:53:58.794Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"de013e87-e4ba-4edd-8f39-83129c3dbbeb","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 15:51:40 +0000 UTC","Now":"2026-07-13T15:53:58.794Z","ShutdownTime":"2026-07-13T15:51:40.000Z"}
{"level":"info","ts":"2026-07-13T15:53:58.794Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"685ceccd-3c58-40c0-87ff-5ec0cd40020b","rayCluster":"fail-pbfxx","podIPs":["10.133.0.20","10.134.0.26"]}
{"level":"info","ts":"2026-07-13T15:53:58.798Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"979270d3-af43-4cf5-84a3-b9b6316b31c2","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:53:58.800Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"b5bfdaeb-3810-4abe-980e-8d5676f71447"}
{"level":"info","ts":"2026-07-13T15:53:58.800Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"b5bfdaeb-3810-4abe-980e-8d5676f71447"}
{"level":"info","ts":"2026-07-13T15:53:58.800Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"b5bfdaeb-3810-4abe-980e-8d5676f71447","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:58.800Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"b5bfdaeb-3810-4abe-980e-8d5676f71447","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:58.800Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"b5bfdaeb-3810-4abe-980e-8d5676f71447","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:58.800Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"b5bfdaeb-3810-4abe-980e-8d5676f71447","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T15:53:58.800Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"b5bfdaeb-3810-4abe-980e-8d5676f71447","podName":"fail-pbfxx-small-group-worker-bhr2g","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-13T15:53:58.800Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"b5bfdaeb-3810-4abe-980e-8d5676f71447","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:53:58.800Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"b5bfdaeb-3810-4abe-980e-8d5676f71447","seconds":300}
{"level":"info","ts":"2026-07-13T15:53:58.807Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"e7dde90a-ad9e-408c-9c58-eb4e40d78a0a"}
{"level":"info","ts":"2026-07-13T15:53:58.807Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"e7dde90a-ad9e-408c-9c58-eb4e40d78a0a"}
{"level":"info","ts":"2026-07-13T15:53:58.807Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"e7dde90a-ad9e-408c-9c58-eb4e40d78a0a","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:58.807Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"e7dde90a-ad9e-408c-9c58-eb4e40d78a0a","Found 1 head Pod":"long-running-qtcvg-head-44cnb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:58.807Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"e7dde90a-ad9e-408c-9c58-eb4e40d78a0a","head Pod":"long-running-qtcvg-head-44cnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qtcvg-head-44cnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:58.807Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"e7dde90a-ad9e-408c-9c58-eb4e40d78a0a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:58.807Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"e7dde90a-ad9e-408c-9c58-eb4e40d78a0a","worker Pod":"long-running-qtcvg-small-group-worker-4jbht","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qtcvg-small-group-worker-4jbht. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:58.807Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"e7dde90a-ad9e-408c-9c58-eb4e40d78a0a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:58.807Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"e7dde90a-ad9e-408c-9c58-eb4e40d78a0a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:53:58.807Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"e7dde90a-ad9e-408c-9c58-eb4e40d78a0a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:58.807Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"e7dde90a-ad9e-408c-9c58-eb4e40d78a0a","podName":"long-running-qtcvg-head-44cnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:58.807Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"e7dde90a-ad9e-408c-9c58-eb4e40d78a0a","podName":"long-running-qtcvg-head-44cnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:53:58.808Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"e7dde90a-ad9e-408c-9c58-eb4e40d78a0a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:53:58.808Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"e7dde90a-ad9e-408c-9c58-eb4e40d78a0a","seconds":300}
{"level":"info","ts":"2026-07-13T15:53:58.962Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"016ddb09-7028-403e-8a67-d38ea1cee532"}
{"level":"info","ts":"2026-07-13T15:53:58.963Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"016ddb09-7028-403e-8a67-d38ea1cee532"}
{"level":"info","ts":"2026-07-13T15:53:58.963Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"016ddb09-7028-403e-8a67-d38ea1cee532","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:58.963Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"016ddb09-7028-403e-8a67-d38ea1cee532","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:58.963Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"016ddb09-7028-403e-8a67-d38ea1cee532","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:58.963Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"016ddb09-7028-403e-8a67-d38ea1cee532","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T15:53:58.963Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"016ddb09-7028-403e-8a67-d38ea1cee532","podName":"fail-pbfxx-small-group-worker-bhr2g","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-13T15:53:58.963Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"016ddb09-7028-403e-8a67-d38ea1cee532","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:53:58.963Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"016ddb09-7028-403e-8a67-d38ea1cee532","seconds":300}
{"level":"info","ts":"2026-07-13T15:53:58.970Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"94f5a46b-1361-4aaf-b5db-8825e51aeae3"}
{"level":"info","ts":"2026-07-13T15:53:58.970Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"94f5a46b-1361-4aaf-b5db-8825e51aeae3"}
{"level":"info","ts":"2026-07-13T15:53:58.970Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"94f5a46b-1361-4aaf-b5db-8825e51aeae3","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:58.970Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"94f5a46b-1361-4aaf-b5db-8825e51aeae3","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:58.970Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"94f5a46b-1361-4aaf-b5db-8825e51aeae3","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:58.970Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"94f5a46b-1361-4aaf-b5db-8825e51aeae3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:58.970Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"94f5a46b-1361-4aaf-b5db-8825e51aeae3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:58.970Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"94f5a46b-1361-4aaf-b5db-8825e51aeae3","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T15:53:58.970Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"94f5a46b-1361-4aaf-b5db-8825e51aeae3","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:58.970Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"94f5a46b-1361-4aaf-b5db-8825e51aeae3","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T15:53:58.970Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"94f5a46b-1361-4aaf-b5db-8825e51aeae3"}
{"level":"info","ts":"2026-07-13T15:53:58.970Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"94f5a46b-1361-4aaf-b5db-8825e51aeae3","nodeType":"worker","rayStartParams":{"address":"fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T15:53:58.970Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"94f5a46b-1361-4aaf-b5db-8825e51aeae3","rayStartCmd":"ray start  --address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:53:58.970Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"94f5a46b-1361-4aaf-b5db-8825e51aeae3","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:53:58.970Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"94f5a46b-1361-4aaf-b5db-8825e51aeae3","enabled":true}
{"level":"error","ts":"2026-07-13T15:53:58.979Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"94f5a46b-1361-4aaf-b5db-8825e51aeae3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:58.979Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"94f5a46b-1361-4aaf-b5db-8825e51aeae3","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:53:58.979Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"94f5a46b-1361-4aaf-b5db-8825e51aeae3","old conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:50:24Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:50:24Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T15:53:58Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}]}
{"level":"info","ts":"2026-07-13T15:53:58.979Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"94f5a46b-1361-4aaf-b5db-8825e51aeae3","name":"fail-pbfxx","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:58Z","stateTransitionTimes":{"ready":"2026-07-13T15:51:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:50:24Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:58Z","stateTransitionTimes":{"ready":"2026-07-13T15:51:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:50:24Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T15:53:58Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:53:59.011Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"94f5a46b-1361-4aaf-b5db-8825e51aeae3"}
{"level":"error","ts":"2026-07-13T15:53:59.011Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"94f5a46b-1361-4aaf-b5db-8825e51aeae3","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:59.012Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"d3c801a2-fc1b-48c6-a289-7325d0616a54","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:53:59.012Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"f23ba432-2e61-4020-ae0b-74016e2651cb","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:53:59.012Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"d3c801a2-fc1b-48c6-a289-7325d0616a54","pod":"fail-pbfxx-head-27cb9","ip":"10.133.0.20"}
{"level":"info","ts":"2026-07-13T15:53:59.012Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"27588cdd-d39c-4280-a159-2c6ce00a4dde","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:59.012Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"27588cdd-d39c-4280-a159-2c6ce00a4dde","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 15:51:40 +0000 UTC","Now":"2026-07-13T15:53:59.012Z","ShutdownTime":"2026-07-13T15:51:40.000Z"}
{"level":"info","ts":"2026-07-13T15:53:59.017Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"bc8c204c-f11c-4d36-ba99-978fca07dcda"}
{"level":"info","ts":"2026-07-13T15:53:59.017Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"bc8c204c-f11c-4d36-ba99-978fca07dcda"}
{"level":"info","ts":"2026-07-13T15:53:59.017Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"bc8c204c-f11c-4d36-ba99-978fca07dcda","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:59.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"bc8c204c-f11c-4d36-ba99-978fca07dcda","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:59.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"bc8c204c-f11c-4d36-ba99-978fca07dcda","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:59.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"bc8c204c-f11c-4d36-ba99-978fca07dcda","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:59.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"bc8c204c-f11c-4d36-ba99-978fca07dcda","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:59.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"bc8c204c-f11c-4d36-ba99-978fca07dcda","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T15:53:59.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"bc8c204c-f11c-4d36-ba99-978fca07dcda","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:59.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"bc8c204c-f11c-4d36-ba99-978fca07dcda","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T15:53:59.017Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"bc8c204c-f11c-4d36-ba99-978fca07dcda"}
{"level":"info","ts":"2026-07-13T15:53:59.017Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"bc8c204c-f11c-4d36-ba99-978fca07dcda","nodeType":"worker","rayStartParams":{"address":"fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T15:53:59.017Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"bc8c204c-f11c-4d36-ba99-978fca07dcda","rayStartCmd":"ray start  --address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:53:59.017Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"bc8c204c-f11c-4d36-ba99-978fca07dcda","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:53:59.017Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"bc8c204c-f11c-4d36-ba99-978fca07dcda","enabled":true}
{"level":"info","ts":"2026-07-13T15:53:59.019Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"f23ba432-2e61-4020-ae0b-74016e2651cb","cluster":"fail-pbfxx"}
{"level":"error","ts":"2026-07-13T15:53:59.021Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"bc8c204c-f11c-4d36-ba99-978fca07dcda","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:59.021Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"bc8c204c-f11c-4d36-ba99-978fca07dcda","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:53:59.021Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"bc8c204c-f11c-4d36-ba99-978fca07dcda"}
{"level":"error","ts":"2026-07-13T15:53:59.022Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"bc8c204c-f11c-4d36-ba99-978fca07dcda","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:59.032Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8ac562e5-237c-4714-b4bf-ed45f89da90d"}
{"level":"info","ts":"2026-07-13T15:53:59.032Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8ac562e5-237c-4714-b4bf-ed45f89da90d"}
{"level":"info","ts":"2026-07-13T15:53:59.032Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8ac562e5-237c-4714-b4bf-ed45f89da90d","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:59.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8ac562e5-237c-4714-b4bf-ed45f89da90d","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:59.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8ac562e5-237c-4714-b4bf-ed45f89da90d","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:59.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8ac562e5-237c-4714-b4bf-ed45f89da90d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:59.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8ac562e5-237c-4714-b4bf-ed45f89da90d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:59.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8ac562e5-237c-4714-b4bf-ed45f89da90d","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T15:53:59.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8ac562e5-237c-4714-b4bf-ed45f89da90d","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:59.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8ac562e5-237c-4714-b4bf-ed45f89da90d","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T15:53:59.032Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8ac562e5-237c-4714-b4bf-ed45f89da90d"}
{"level":"info","ts":"2026-07-13T15:53:59.032Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8ac562e5-237c-4714-b4bf-ed45f89da90d","nodeType":"worker","rayStartParams":{"address":"fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T15:53:59.032Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8ac562e5-237c-4714-b4bf-ed45f89da90d","rayStartCmd":"ray start  --address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:53:59.032Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8ac562e5-237c-4714-b4bf-ed45f89da90d","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:53:59.032Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8ac562e5-237c-4714-b4bf-ed45f89da90d","enabled":true}
{"level":"info","ts":"2026-07-13T15:53:59.036Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"d3c801a2-fc1b-48c6-a289-7325d0616a54","certificate":"ray-head-cert-fail-pbfxx","dnsNames":["fail-pbfxx-head-svc","localhost","fail-pbfxx-head-svc.test-ns-5xvz7.svc","fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local"],"ipAddresses":["10.133.0.20","127.0.0.1"]}
{"level":"error","ts":"2026-07-13T15:53:59.037Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8ac562e5-237c-4714-b4bf-ed45f89da90d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:59.037Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8ac562e5-237c-4714-b4bf-ed45f89da90d","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:53:59.037Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8ac562e5-237c-4714-b4bf-ed45f89da90d"}
{"level":"error","ts":"2026-07-13T15:53:59.037Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8ac562e5-237c-4714-b4bf-ed45f89da90d","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:59.053Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"d3c801a2-fc1b-48c6-a289-7325d0616a54","certificate":"ray-worker-cert-fail-pbfxx","dnsNames":["fail-pbfxx-worker-svc","localhost","fail-pbfxx-small-group","fail-pbfxx-worker-svc.test-ns-5xvz7.svc","fail-pbfxx-worker-svc.test-ns-5xvz7.svc.cluster.local","*.fail-pbfxx-worker-svc.test-ns-5xvz7.svc","*.fail-pbfxx-worker-svc.test-ns-5xvz7.svc.cluster.local","*-worker-*.test-ns-5xvz7.svc","*-worker-*.test-ns-5xvz7.svc.cluster.local"],"ipAddresses":["10.133.0.20","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:53:59.053Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"d3c801a2-fc1b-48c6-a289-7325d0616a54","rayCluster":"fail-pbfxx","podIPs":["10.133.0.20"]}
{"level":"info","ts":"2026-07-13T15:53:59.057Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"11ac476c-2c2e-4fdb-a3a7-c650c45c12bc"}
{"level":"info","ts":"2026-07-13T15:53:59.057Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"11ac476c-2c2e-4fdb-a3a7-c650c45c12bc"}
{"level":"info","ts":"2026-07-13T15:53:59.057Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"11ac476c-2c2e-4fdb-a3a7-c650c45c12bc","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:59.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"11ac476c-2c2e-4fdb-a3a7-c650c45c12bc","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:59.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"11ac476c-2c2e-4fdb-a3a7-c650c45c12bc","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:59.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"11ac476c-2c2e-4fdb-a3a7-c650c45c12bc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:59.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"11ac476c-2c2e-4fdb-a3a7-c650c45c12bc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:59.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"11ac476c-2c2e-4fdb-a3a7-c650c45c12bc","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T15:53:59.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"11ac476c-2c2e-4fdb-a3a7-c650c45c12bc","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:59.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"11ac476c-2c2e-4fdb-a3a7-c650c45c12bc","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T15:53:59.058Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"11ac476c-2c2e-4fdb-a3a7-c650c45c12bc"}
{"level":"info","ts":"2026-07-13T15:53:59.058Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"11ac476c-2c2e-4fdb-a3a7-c650c45c12bc","nodeType":"worker","rayStartParams":{"address":"fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T15:53:59.058Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"11ac476c-2c2e-4fdb-a3a7-c650c45c12bc","rayStartCmd":"ray start  --address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:53:59.058Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"11ac476c-2c2e-4fdb-a3a7-c650c45c12bc","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:53:59.058Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"11ac476c-2c2e-4fdb-a3a7-c650c45c12bc","enabled":true}
{"level":"error","ts":"2026-07-13T15:53:59.061Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"11ac476c-2c2e-4fdb-a3a7-c650c45c12bc","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:59.061Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"11ac476c-2c2e-4fdb-a3a7-c650c45c12bc","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:53:59.061Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"11ac476c-2c2e-4fdb-a3a7-c650c45c12bc"}
{"level":"error","ts":"2026-07-13T15:53:59.061Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"11ac476c-2c2e-4fdb-a3a7-c650c45c12bc","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:59.102Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8dad34c0-20d5-4253-a25a-aaac31dbeacb"}
{"level":"info","ts":"2026-07-13T15:53:59.102Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8dad34c0-20d5-4253-a25a-aaac31dbeacb"}
{"level":"info","ts":"2026-07-13T15:53:59.102Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8dad34c0-20d5-4253-a25a-aaac31dbeacb","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:59.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8dad34c0-20d5-4253-a25a-aaac31dbeacb","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:59.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8dad34c0-20d5-4253-a25a-aaac31dbeacb","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:59.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8dad34c0-20d5-4253-a25a-aaac31dbeacb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:59.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8dad34c0-20d5-4253-a25a-aaac31dbeacb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:59.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8dad34c0-20d5-4253-a25a-aaac31dbeacb","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T15:53:59.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8dad34c0-20d5-4253-a25a-aaac31dbeacb","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:59.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8dad34c0-20d5-4253-a25a-aaac31dbeacb","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T15:53:59.102Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8dad34c0-20d5-4253-a25a-aaac31dbeacb"}
{"level":"info","ts":"2026-07-13T15:53:59.102Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8dad34c0-20d5-4253-a25a-aaac31dbeacb","nodeType":"worker","rayStartParams":{"address":"fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T15:53:59.102Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8dad34c0-20d5-4253-a25a-aaac31dbeacb","rayStartCmd":"ray start  --address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:53:59.102Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8dad34c0-20d5-4253-a25a-aaac31dbeacb","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:53:59.102Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8dad34c0-20d5-4253-a25a-aaac31dbeacb","enabled":true}
{"level":"error","ts":"2026-07-13T15:53:59.106Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8dad34c0-20d5-4253-a25a-aaac31dbeacb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:59.106Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8dad34c0-20d5-4253-a25a-aaac31dbeacb","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:53:59.106Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8dad34c0-20d5-4253-a25a-aaac31dbeacb"}
{"level":"error","ts":"2026-07-13T15:53:59.106Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"8dad34c0-20d5-4253-a25a-aaac31dbeacb","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:59.187Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"600aa517-94ab-44fd-a325-7992b5d54f1b"}
{"level":"info","ts":"2026-07-13T15:53:59.187Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"600aa517-94ab-44fd-a325-7992b5d54f1b"}
{"level":"info","ts":"2026-07-13T15:53:59.187Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"600aa517-94ab-44fd-a325-7992b5d54f1b","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:59.187Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"600aa517-94ab-44fd-a325-7992b5d54f1b","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:53:59.187Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"600aa517-94ab-44fd-a325-7992b5d54f1b","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-pbfxx-head-27cb9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:53:59.187Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"600aa517-94ab-44fd-a325-7992b5d54f1b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:59.188Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"600aa517-94ab-44fd-a325-7992b5d54f1b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:59.188Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"600aa517-94ab-44fd-a325-7992b5d54f1b","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T15:53:59.188Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"600aa517-94ab-44fd-a325-7992b5d54f1b","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:59.188Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"600aa517-94ab-44fd-a325-7992b5d54f1b","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T15:53:59.188Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"600aa517-94ab-44fd-a325-7992b5d54f1b"}
{"level":"info","ts":"2026-07-13T15:53:59.188Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"600aa517-94ab-44fd-a325-7992b5d54f1b","nodeType":"worker","rayStartParams":{"address":"fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T15:53:59.188Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"600aa517-94ab-44fd-a325-7992b5d54f1b","rayStartCmd":"ray start  --address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:53:59.188Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"600aa517-94ab-44fd-a325-7992b5d54f1b","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:53:59.188Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"600aa517-94ab-44fd-a325-7992b5d54f1b","enabled":true}
{"level":"error","ts":"2026-07-13T15:53:59.191Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"600aa517-94ab-44fd-a325-7992b5d54f1b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:59.191Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"600aa517-94ab-44fd-a325-7992b5d54f1b","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:53:59.191Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"600aa517-94ab-44fd-a325-7992b5d54f1b"}
{"level":"error","ts":"2026-07-13T15:53:59.191Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"600aa517-94ab-44fd-a325-7992b5d54f1b","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:59.280Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"156e4094-a2aa-43a5-93ae-faf5a0466fbe"}
{"level":"info","ts":"2026-07-13T15:53:59.280Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"156e4094-a2aa-43a5-93ae-faf5a0466fbe"}
{"level":"info","ts":"2026-07-13T15:53:59.280Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"156e4094-a2aa-43a5-93ae-faf5a0466fbe","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:59.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"156e4094-a2aa-43a5-93ae-faf5a0466fbe","Found 1 head Pod":"fail-pbfxx-head-27cb9","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-07-13 15:50:12 +0000 UTC,FinishedAt:2026-07-13 15:53:59 +0000 UTC,ContainerID:cri-o://f924762a701e9e81accb35f7f479fc2fd876eb76fe1a01ddec84c80d38b753a8,}"}
{"level":"info","ts":"2026-07-13T15:53:59.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"156e4094-a2aa-43a5-93ae-faf5a0466fbe","head Pod":"fail-pbfxx-head-27cb9","shouldDelete":true,"reason":"The head Pod fail-pbfxx-head-27cb9 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-07-13T15:53:59.283Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"156e4094-a2aa-43a5-93ae-faf5a0466fbe","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod fail-pbfxx-head-27cb9 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:59.283Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"156e4094-a2aa-43a5-93ae-faf5a0466fbe","error":"The head Pod fail-pbfxx-head-27cb9 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-07-13T15:53:59.284Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"156e4094-a2aa-43a5-93ae-faf5a0466fbe","old conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:50:24Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T15:53:58Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:59Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T15:53:58Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}]}
{"level":"info","ts":"2026-07-13T15:53:59.284Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"156e4094-a2aa-43a5-93ae-faf5a0466fbe","name":"fail-pbfxx","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:58Z","stateTransitionTimes":{"ready":"2026-07-13T15:51:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:50:24Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T15:53:58Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:59Z","stateTransitionTimes":{"ready":"2026-07-13T15:51:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:59Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T15:53:58Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:53:59.296Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"156e4094-a2aa-43a5-93ae-faf5a0466fbe"}
{"level":"error","ts":"2026-07-13T15:53:59.296Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"156e4094-a2aa-43a5-93ae-faf5a0466fbe","error":"The head Pod fail-pbfxx-head-27cb9 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:59.296Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"35628171-97e0-4207-bcac-446c6abb75fd","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:53:59.296Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"7a3ad2dd-33ff-4757-afe7-af98b154387a","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:53:59.296Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"240f9c76-62d4-4ce0-9361-480b317f6623","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:59.296Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"240f9c76-62d4-4ce0-9361-480b317f6623","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 15:51:40 +0000 UTC","Now":"2026-07-13T15:53:59.296Z","ShutdownTime":"2026-07-13T15:51:40.000Z"}
{"level":"info","ts":"2026-07-13T15:53:59.296Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"35628171-97e0-4207-bcac-446c6abb75fd","pod":"fail-pbfxx-head-27cb9","ip":"10.133.0.20"}
{"level":"info","ts":"2026-07-13T15:53:59.297Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"35628171-97e0-4207-bcac-446c6abb75fd"}
{"level":"info","ts":"2026-07-13T15:53:59.301Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"7a3ad2dd-33ff-4757-afe7-af98b154387a","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:53:59.322Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"85efd2e4-c8d4-4482-8f51-ac2aa37511bd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:53:59.322Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"85efd2e4-c8d4-4482-8f51-ac2aa37511bd","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:53:59.322Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"85efd2e4-c8d4-4482-8f51-ac2aa37511bd","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:53:59.322Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"85efd2e4-c8d4-4482-8f51-ac2aa37511bd"}
{"level":"info","ts":"2026-07-13T15:53:59.322Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"85efd2e4-c8d4-4482-8f51-ac2aa37511bd","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:53:59.352Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c735a585-cc39-4a05-ac4a-93956dde3b0f"}
{"level":"info","ts":"2026-07-13T15:53:59.352Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c735a585-cc39-4a05-ac4a-93956dde3b0f"}
{"level":"info","ts":"2026-07-13T15:53:59.352Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c735a585-cc39-4a05-ac4a-93956dde3b0f","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:59.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c735a585-cc39-4a05-ac4a-93956dde3b0f","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-13T15:53:59.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c735a585-cc39-4a05-ac4a-93956dde3b0f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:59.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c735a585-cc39-4a05-ac4a-93956dde3b0f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:59.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c735a585-cc39-4a05-ac4a-93956dde3b0f","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T15:53:59.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c735a585-cc39-4a05-ac4a-93956dde3b0f","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:59.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c735a585-cc39-4a05-ac4a-93956dde3b0f","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T15:53:59.352Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c735a585-cc39-4a05-ac4a-93956dde3b0f"}
{"level":"info","ts":"2026-07-13T15:53:59.352Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c735a585-cc39-4a05-ac4a-93956dde3b0f","nodeType":"worker","rayStartParams":{"address":"fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T15:53:59.352Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c735a585-cc39-4a05-ac4a-93956dde3b0f","rayStartCmd":"ray start  --address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:53:59.352Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c735a585-cc39-4a05-ac4a-93956dde3b0f","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:53:59.352Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c735a585-cc39-4a05-ac4a-93956dde3b0f","enabled":true}
{"level":"error","ts":"2026-07-13T15:53:59.356Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c735a585-cc39-4a05-ac4a-93956dde3b0f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:59.356Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c735a585-cc39-4a05-ac4a-93956dde3b0f","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:53:59.356Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c735a585-cc39-4a05-ac4a-93956dde3b0f"}
{"level":"error","ts":"2026-07-13T15:53:59.356Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"c735a585-cc39-4a05-ac4a-93956dde3b0f","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:59.806Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916"}
{"level":"info","ts":"2026-07-13T15:53:59.806Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916"}
{"level":"info","ts":"2026-07-13T15:53:59.806Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:59.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-13T15:53:59.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:53:59.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:59.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T15:53:59.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T15:53:59.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T15:53:59.807Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916"}
{"level":"info","ts":"2026-07-13T15:53:59.807Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916","nodeType":"worker","rayStartParams":{"address":"fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T15:53:59.807Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916","rayStartCmd":"ray start  --address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:53:59.807Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:53:59.807Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916","enabled":true}
{"level":"error","ts":"2026-07-13T15:53:59.813Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:59.813Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:53:59.813Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:59.813Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:59.813Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:59.813Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"newHead":{"serviceName":"fail-pbfxx-head-svc"}}
{"level":"info","ts":"2026-07-13T15:53:59.813Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916","name":"fail-pbfxx","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:59Z","stateTransitionTimes":{"ready":"2026-07-13T15:51:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:59Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T15:53:58Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:59Z","stateTransitionTimes":{"ready":"2026-07-13T15:51:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T15:53:58Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:53:59.825Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916"}
{"level":"error","ts":"2026-07-13T15:53:59.825Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"5be13bad-1e61-42d7-95eb-07eff852d916","error":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:59.825Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859"}
{"level":"info","ts":"2026-07-13T15:53:59.825Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859"}
{"level":"info","ts":"2026-07-13T15:53:59.825Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:59.825Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859"}
{"level":"info","ts":"2026-07-13T15:53:59.825Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:53:59.825Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859"}
{"level":"info","ts":"2026-07-13T15:53:59.825Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859","mode":"IntegratedOAuth","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:53:59.825Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859","cluster":"fail-pbfxx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:53:59.825Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859","cluster":"fail-pbfxx","authType":"IntegratedOAuth","serviceAccount":"fail-pbfxx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T15:53:59.825Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-pbfxx","ray.io/group":"headgroup","ray.io/identifier":"fail-pbfxx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T15:53:59.825Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T15:53:59.825Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:53:59.825Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:53:59.825Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859","enabled":true}
{"level":"info","ts":"2026-07-13T15:53:59.826Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"04d39177-992c-43ce-8f8b-3a83c97a5878","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:53:59.826Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"a309bcbf-7511-4123-8978-a8822838fe5f","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:53:59.826Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"7afb9a84-aa9e-4fe5-83b4-6bb5f7577ece","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:53:59.826Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"7afb9a84-aa9e-4fe5-83b4-6bb5f7577ece","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 15:51:40 +0000 UTC","Now":"2026-07-13T15:53:59.826Z","ShutdownTime":"2026-07-13T15:51:40.000Z"}
{"level":"error","ts":"2026-07-13T15:53:59.828Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-pbfxx-head-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:59.828Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859","error":"FailedCreateHeadPod\npods \"fail-pbfxx-head-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:53:59.828Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:59.828Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:59.828Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:53:59.828Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"newHead":{"serviceName":"fail-pbfxx-head-svc"}}
{"level":"info","ts":"2026-07-13T15:53:59.828Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859","name":"fail-pbfxx","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:59Z","stateTransitionTimes":{"ready":"2026-07-13T15:51:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.20","serviceIP":"10.133.0.20","podName":"fail-pbfxx-head-27cb9","serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:59Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T15:53:58Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:59Z","stateTransitionTimes":{"ready":"2026-07-13T15:51:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T15:53:58Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"fail-pbfxx-head-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:53:59.830Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"a309bcbf-7511-4123-8978-a8822838fe5f","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:53:59.836Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859","name":"fail-pbfxx","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-pbfxx\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-5xvz7","name":"fail-pbfxx"}}
{"level":"info","ts":"2026-07-13T15:53:59.836Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859"}
{"level":"error","ts":"2026-07-13T15:53:59.836Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"2dd9d8e5-3e79-4187-8013-886618ed3859","error":"FailedCreateHeadPod\npods \"fail-pbfxx-head-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:53:59.838Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"04d39177-992c-43ce-8f8b-3a83c97a5878","certificate":"ray-head-cert-fail-pbfxx","dnsNames":["fail-pbfxx-head-svc","localhost","fail-pbfxx-head-svc.test-ns-5xvz7.svc","fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:53:59.850Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"04d39177-992c-43ce-8f8b-3a83c97a5878","certificate":"ray-worker-cert-fail-pbfxx","dnsNames":["fail-pbfxx-worker-svc","localhost","fail-pbfxx-small-group","fail-pbfxx-worker-svc.test-ns-5xvz7.svc","fail-pbfxx-worker-svc.test-ns-5xvz7.svc.cluster.local","*.fail-pbfxx-worker-svc.test-ns-5xvz7.svc","*.fail-pbfxx-worker-svc.test-ns-5xvz7.svc.cluster.local","*-worker-*.test-ns-5xvz7.svc","*-worker-*.test-ns-5xvz7.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:53:59.850Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"04d39177-992c-43ce-8f8b-3a83c97a5878"}
{"level":"info","ts":"2026-07-13T15:53:59.997Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a"}
{"level":"info","ts":"2026-07-13T15:53:59.997Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a"}
{"level":"info","ts":"2026-07-13T15:53:59.997Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a","1 head service found":"fail-pbfxx-head-svc"}
{"level":"info","ts":"2026-07-13T15:53:59.997Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a"}
{"level":"info","ts":"2026-07-13T15:53:59.997Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:53:59.997Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a"}
{"level":"info","ts":"2026-07-13T15:53:59.997Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a","mode":"IntegratedOAuth","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:53:59.997Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a","cluster":"fail-pbfxx","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:53:59.997Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a","cluster":"fail-pbfxx","authType":"IntegratedOAuth","serviceAccount":"fail-pbfxx-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T15:53:59.997Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-pbfxx","ray.io/group":"headgroup","ray.io/identifier":"fail-pbfxx-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T15:53:59.997Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T15:53:59.997Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:53:59.997Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-pbfxx-head-svc.test-ns-5xvz7.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:53:59.997Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a","enabled":true}
{"level":"error","ts":"2026-07-13T15:54:00.000Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-pbfxx-head-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:54:00.000Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a","error":"FailedCreateHeadPod\npods \"fail-pbfxx-head-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}
{"level":"info","ts":"2026-07-13T15:54:00.000Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:54:00.001Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:54:00.001Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a","filter labels":["test-ns-5xvz7",{"ray.io/cluster":"fail-pbfxx","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:54:00.001Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T15:53:58Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T15:53:58Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"fail-pbfxx-head-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}]}
{"level":"info","ts":"2026-07-13T15:54:00.001Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a","name":"fail-pbfxx","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:59Z","stateTransitionTimes":{"ready":"2026-07-13T15:51:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T15:53:58Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-pbfxx-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:54:00Z","stateTransitionTimes":{"ready":"2026-07-13T15:51:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-pbfxx-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:51:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:50:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:50:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T15:53:58Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"fail-pbfxx-head-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:54:00.012Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a"}
{"level":"error","ts":"2026-07-13T15:54:00.012Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"33a436e5-7c8c-4cfb-86a8-7fcc94f7e56a","error":"FailedCreateHeadPod\npods \"fail-pbfxx-head-\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:54:00.013Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"0777da59-d92d-42c6-9c88-3c0ff36df350","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:54:00.013Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"b5d1a622-d448-49b3-b8f0-b7a63445a3b4","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:54:00.013Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"0777da59-d92d-42c6-9c88-3c0ff36df350"}
{"level":"info","ts":"2026-07-13T15:54:00.013Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"32d1171f-5611-4bb4-a2d4-a0c5ac526081","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-07-13T15:54:00.013Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"32d1171f-5611-4bb4-a2d4-a0c5ac526081","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 15:51:40 +0000 UTC","Now":"2026-07-13T15:54:00.013Z","ShutdownTime":"2026-07-13T15:51:40.000Z"}
{"level":"info","ts":"2026-07-13T15:54:00.017Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"b5d1a622-d448-49b3-b8f0-b7a63445a3b4","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:54:01.821Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872f3683-c4a4-40d6-8bb0-5608d8f591a4"}
{"level":"info","ts":"2026-07-13T15:54:01.821Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872f3683-c4a4-40d6-8bb0-5608d8f591a4"}
{"level":"info","ts":"2026-07-13T15:54:01.821Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872f3683-c4a4-40d6-8bb0-5608d8f591a4","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:54:01.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872f3683-c4a4-40d6-8bb0-5608d8f591a4","Found 1 head Pod":"long-running-qtcvg-head-44cnb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:54:01.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872f3683-c4a4-40d6-8bb0-5608d8f591a4","head Pod":"long-running-qtcvg-head-44cnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qtcvg-head-44cnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:54:01.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872f3683-c4a4-40d6-8bb0-5608d8f591a4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:54:01.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872f3683-c4a4-40d6-8bb0-5608d8f591a4","worker Pod":"long-running-qtcvg-small-group-worker-4jbht","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qtcvg-small-group-worker-4jbht. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:54:01.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872f3683-c4a4-40d6-8bb0-5608d8f591a4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:54:01.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872f3683-c4a4-40d6-8bb0-5608d8f591a4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:54:01.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872f3683-c4a4-40d6-8bb0-5608d8f591a4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:54:01.821Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872f3683-c4a4-40d6-8bb0-5608d8f591a4","podName":"long-running-qtcvg-head-44cnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:54:01.821Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872f3683-c4a4-40d6-8bb0-5608d8f591a4","podName":"long-running-qtcvg-head-44cnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:54:01.821Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872f3683-c4a4-40d6-8bb0-5608d8f591a4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:54:01.821Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"872f3683-c4a4-40d6-8bb0-5608d8f591a4","seconds":300}
{"level":"info","ts":"2026-07-13T15:54:02.322Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"658397ef-9b7f-4dbd-b15b-b298de2d4123","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:02.322Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"658397ef-9b7f-4dbd-b15b-b298de2d4123","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:02.322Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"658397ef-9b7f-4dbd-b15b-b298de2d4123","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:02.323Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"658397ef-9b7f-4dbd-b15b-b298de2d4123"}
{"level":"info","ts":"2026-07-13T15:54:02.323Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"658397ef-9b7f-4dbd-b15b-b298de2d4123","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:54:02.828Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"05109e2c-e50d-4359-a779-d224cf53e161"}
{"level":"info","ts":"2026-07-13T15:54:02.828Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"05109e2c-e50d-4359-a779-d224cf53e161"}
{"level":"info","ts":"2026-07-13T15:54:02.829Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"05109e2c-e50d-4359-a779-d224cf53e161","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:54:02.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"05109e2c-e50d-4359-a779-d224cf53e161","Found 1 head Pod":"long-running-qtcvg-head-44cnb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:54:02.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"05109e2c-e50d-4359-a779-d224cf53e161","head Pod":"long-running-qtcvg-head-44cnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qtcvg-head-44cnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:54:02.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"05109e2c-e50d-4359-a779-d224cf53e161","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:54:02.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"05109e2c-e50d-4359-a779-d224cf53e161","worker Pod":"long-running-qtcvg-small-group-worker-4jbht","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qtcvg-small-group-worker-4jbht. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:54:02.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"05109e2c-e50d-4359-a779-d224cf53e161","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:54:02.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"05109e2c-e50d-4359-a779-d224cf53e161","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:54:02.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"05109e2c-e50d-4359-a779-d224cf53e161","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:54:02.829Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"05109e2c-e50d-4359-a779-d224cf53e161","podName":"long-running-qtcvg-head-44cnb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:53:56Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:54:02.829Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"05109e2c-e50d-4359-a779-d224cf53e161","podName":"long-running-qtcvg-head-44cnb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:53:56Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:54:02.829Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"05109e2c-e50d-4359-a779-d224cf53e161","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:54:02.829Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"05109e2c-e50d-4359-a779-d224cf53e161","name":"long-running-qtcvg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:53:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.22","serviceIP":"10.133.0.22","podName":"long-running-qtcvg-head-44cnb","serviceName":"long-running-qtcvg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:54:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.22","serviceIP":"10.133.0.22","podName":"long-running-qtcvg-head-44cnb","serviceName":"long-running-qtcvg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:54:02.841Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"cdc23cf0-b149-4c84-a02e-562f8deea5d4","rayCluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:02.841Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"8739d075-2384-4c23-98f7-2db73d5f9a98","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:02.841Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"cdc23cf0-b149-4c84-a02e-562f8deea5d4","pod":"long-running-qtcvg-head-44cnb","ip":"10.133.0.22"}
{"level":"info","ts":"2026-07-13T15:54:02.841Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"cdc23cf0-b149-4c84-a02e-562f8deea5d4","pod":"long-running-qtcvg-small-group-worker-4jbht","ip":"10.134.0.28"}
{"level":"info","ts":"2026-07-13T15:54:02.841Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"62ed7b27-5719-4e8f-9a9c-b5668998e8d5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:02.841Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"62ed7b27-5719-4e8f-9a9c-b5668998e8d5","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:02.841Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"62ed7b27-5719-4e8f-9a9c-b5668998e8d5","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:02.841Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"62ed7b27-5719-4e8f-9a9c-b5668998e8d5"}
{"level":"info","ts":"2026-07-13T15:54:02.841Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"62ed7b27-5719-4e8f-9a9c-b5668998e8d5","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:54:02.852Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"8739d075-2384-4c23-98f7-2db73d5f9a98","name":"long-running-qtcvg-head"}
{"level":"info","ts":"2026-07-13T15:54:02.852Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"cdc23cf0-b149-4c84-a02e-562f8deea5d4","certificate":"ray-head-cert-long-running-qtcvg","dnsNames":["long-running-qtcvg-head-svc","localhost","long-running-qtcvg-head-svc.test-ns-88gnn.svc","long-running-qtcvg-head-svc.test-ns-88gnn.svc.cluster.local"],"ipAddresses":["10.133.0.22","10.134.0.28","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:54:02.921Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"8739d075-2384-4c23-98f7-2db73d5f9a98","name":"long-running-qtcvg-workers"}
{"level":"info","ts":"2026-07-13T15:54:02.921Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"8739d075-2384-4c23-98f7-2db73d5f9a98","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:02.922Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"cdc23cf0-b149-4c84-a02e-562f8deea5d4","certificate":"ray-worker-cert-long-running-qtcvg","dnsNames":["long-running-qtcvg-worker-svc","localhost","long-running-qtcvg-small-group","long-running-qtcvg-worker-svc.test-ns-88gnn.svc","long-running-qtcvg-worker-svc.test-ns-88gnn.svc.cluster.local","*.long-running-qtcvg-worker-svc.test-ns-88gnn.svc","*.long-running-qtcvg-worker-svc.test-ns-88gnn.svc.cluster.local","*-worker-*.test-ns-88gnn.svc","*-worker-*.test-ns-88gnn.svc.cluster.local"],"ipAddresses":["10.133.0.22","10.134.0.28","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:54:02.922Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"cdc23cf0-b149-4c84-a02e-562f8deea5d4","rayCluster":"long-running-qtcvg","podIPs":["10.133.0.22","10.134.0.28"]}
{"level":"info","ts":"2026-07-13T15:54:03.507Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"5b3328be-8b7b-4d57-827a-a5278c87f165","DeletionTimestamp":"2026-07-13 15:54:03 +0000 UTC"}
{"level":"info","ts":"2026-07-13T15:54:03.507Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"5b3328be-8b7b-4d57-827a-a5278c87f165","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T15:54:03.519Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"8835d3ca-4d52-4b8f-bcbd-c36d6debfe9e","DeletionTimestamp":"2026-07-13 15:54:03 +0000 UTC"}
{"level":"info","ts":"2026-07-13T15:54:03.519Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"8835d3ca-4d52-4b8f-bcbd-c36d6debfe9e","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T15:54:03.530Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"7c9d2cba-1a15-42c2-8443-62da364d8c41"}
{"level":"info","ts":"2026-07-13T15:54:03.530Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"896a0564-a4fa-47e2-add2-b77c019f6938"}
{"level":"info","ts":"2026-07-13T15:54:03.540Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"b07e71db-17d0-4d20-8b33-108b4a8e574a","namespacedName":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:54:03.540Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"9142e6fb-f83e-4917-8bf2-e55fa5eea4cb","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:54:03.540Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"b07e71db-17d0-4d20-8b33-108b4a8e574a","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:54:03.540Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"b07e71db-17d0-4d20-8b33-108b4a8e574a","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:54:03.540Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"9142e6fb-f83e-4917-8bf2-e55fa5eea4cb","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:54:03.540Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"cf8ab554-932a-4b87-8bbb-f3740db5a492"}
{"level":"info","ts":"2026-07-13T15:54:03.540Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"reconcileID":"cb3878d5-0cf1-47dc-9729-de42ba13b93b"}
{"level":"info","ts":"2026-07-13T15:54:03.540Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"68c69706-28f4-4242-bdf6-ac00704b2aa0"}
{"level":"info","ts":"2026-07-13T15:54:03.540Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"b07e71db-17d0-4d20-8b33-108b4a8e574a","namespace":"test-ns-5xvz7","otherAuthClusters":0,"action":"will delete"}
{"level":"error","ts":"2026-07-13T15:54:03.542Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"9142e6fb-f83e-4917-8bf2-e55fa5eea4cb","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-pbfxx\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:54:03.542Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"9142e6fb-f83e-4917-8bf2-e55fa5eea4cb","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-pbfxx\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:54:03.542Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"9142e6fb-f83e-4917-8bf2-e55fa5eea4cb"}
{"level":"error","ts":"2026-07-13T15:54:03.542Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"9142e6fb-f83e-4917-8bf2-e55fa5eea4cb","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-pbfxx\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:54:03.547Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"b07e71db-17d0-4d20-8b33-108b4a8e574a","name":"kuberay-gateway-access","namespace":"test-ns-5xvz7"}
{"level":"info","ts":"2026-07-13T15:54:03.548Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c44d40a1-e8f4-4915-9c28-fbcde60d5c24","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:54:03.548Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c44d40a1-e8f4-4915-9c28-fbcde60d5c24","rayCluster":"fail-pbfxx"}
{"level":"error","ts":"2026-07-13T15:54:03.550Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c44d40a1-e8f4-4915-9c28-fbcde60d5c24","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-pbfxx\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:54:03.550Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c44d40a1-e8f4-4915-9c28-fbcde60d5c24","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-pbfxx\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:54:03.550Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c44d40a1-e8f4-4915-9c28-fbcde60d5c24"}
{"level":"error","ts":"2026-07-13T15:54:03.550Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c44d40a1-e8f4-4915-9c28-fbcde60d5c24","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-pbfxx\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:54:03.552Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"b07e71db-17d0-4d20-8b33-108b4a8e574a","name":"test-ns-5xvz7-fail-pbfxx","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-13T15:54:03.557Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"b07e71db-17d0-4d20-8b33-108b4a8e574a","serviceAccount":"fail-pbfxx-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:54:03.560Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"5cc529f5-2a50-447e-8080-d0cbcc44c846","rayCluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:54:03.561Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"5cc529f5-2a50-447e-8080-d0cbcc44c846","rayCluster":"fail-pbfxx"}
{"level":"error","ts":"2026-07-13T15:54:03.563Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"5cc529f5-2a50-447e-8080-d0cbcc44c846","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-pbfxx\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:54:03.563Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"5cc529f5-2a50-447e-8080-d0cbcc44c846","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-pbfxx\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:54:03.563Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"5cc529f5-2a50-447e-8080-d0cbcc44c846"}
{"level":"error","ts":"2026-07-13T15:54:03.563Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"5cc529f5-2a50-447e-8080-d0cbcc44c846","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-pbfxx\" is forbidden: unable to create new content in namespace test-ns-5xvz7 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:54:03.564Z","logger":"raycluster-resource","msg":"default","name":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:54:03.564Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-pbfxx","namespace":"test-ns-5xvz7"}
{"level":"info","ts":"2026-07-13T15:54:03.572Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"b07e71db-17d0-4d20-8b33-108b4a8e574a","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:54:03.572Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c377a812-9fdb-40b1-8d91-8569aa2445b6","namespacedName":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:54:03.572Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c377a812-9fdb-40b1-8d91-8569aa2445b6","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:54:03.572Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c377a812-9fdb-40b1-8d91-8569aa2445b6","cluster":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:54:03.572Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c377a812-9fdb-40b1-8d91-8569aa2445b6","namespace":"test-ns-5xvz7","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-13T15:54:03.572Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c377a812-9fdb-40b1-8d91-8569aa2445b6","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-13T15:54:03.573Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"79136690-9438-41cf-8495-b367cef7e8a4"}
{"level":"info","ts":"2026-07-13T15:54:03.573Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"bf95ca08-d33b-421d-89be-16a8f8d4707d","namespace":"test-ns-5xvz7","clusterName":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:54:03.573Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"7d398692-c2b6-498e-9e14-be6d38d8fe2d"}
{"level":"error","ts":"2026-07-13T15:54:03.576Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c377a812-9fdb-40b1-8d91-8569aa2445b6","error":"rayclusters.ray.io \"fail-pbfxx\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:54:03.576Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c377a812-9fdb-40b1-8d91-8569aa2445b6"}
{"level":"error","ts":"2026-07-13T15:54:03.576Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c377a812-9fdb-40b1-8d91-8569aa2445b6","error":"rayclusters.ray.io \"fail-pbfxx\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:54:03.576Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"bf6fe829-07a8-4c5b-8299-507f2342143c","namespacedName":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:54:03.576Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"bf6fe829-07a8-4c5b-8299-507f2342143c"}
{"level":"info","ts":"2026-07-13T15:54:03.576Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"bf6fe829-07a8-4c5b-8299-507f2342143c","namespace":"test-ns-5xvz7","authClusters":0}
{"level":"info","ts":"2026-07-13T15:54:03.576Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"bf6fe829-07a8-4c5b-8299-507f2342143c","namespace":"test-ns-5xvz7"}
{"level":"info","ts":"2026-07-13T15:54:03.577Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"3549fe07-1f46-421a-aef0-ba5f4de83a15"}
{"level":"info","ts":"2026-07-13T15:54:03.578Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"355c4c52-dc01-4231-982f-bbcc18c55532"}
{"level":"info","ts":"2026-07-13T15:54:03.579Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c56ed87d-9854-42f8-aa27-c1d2a39b3a0c","namespacedName":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:54:03.579Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c56ed87d-9854-42f8-aa27-c1d2a39b3a0c"}
{"level":"info","ts":"2026-07-13T15:54:03.579Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c56ed87d-9854-42f8-aa27-c1d2a39b3a0c","namespace":"test-ns-5xvz7","authClusters":0}
{"level":"info","ts":"2026-07-13T15:54:03.579Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"c56ed87d-9854-42f8-aa27-c1d2a39b3a0c","namespace":"test-ns-5xvz7"}
{"level":"info","ts":"2026-07-13T15:54:03.581Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"86f67e1e-784d-4de0-8d5d-b95934cf6ad1","namespacedName":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"}}
{"level":"info","ts":"2026-07-13T15:54:03.581Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"86f67e1e-784d-4de0-8d5d-b95934cf6ad1"}
{"level":"info","ts":"2026-07-13T15:54:03.581Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"86f67e1e-784d-4de0-8d5d-b95934cf6ad1","namespace":"test-ns-5xvz7","authClusters":0}
{"level":"info","ts":"2026-07-13T15:54:03.581Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"86f67e1e-784d-4de0-8d5d-b95934cf6ad1","namespace":"test-ns-5xvz7"}
{"level":"info","ts":"2026-07-13T15:54:03.626Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"bf95ca08-d33b-421d-89be-16a8f8d4707d","namespace":"test-ns-5xvz7","clusterName":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:54:03.626Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"16429ff0-9d14-43d2-a53d-bad32c5830f5","namespace":"test-ns-5xvz7","clusterName":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:54:03.626Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-pbfxx","namespace":"test-ns-5xvz7"},"namespace":"test-ns-5xvz7","name":"fail-pbfxx","reconcileID":"16429ff0-9d14-43d2-a53d-bad32c5830f5","namespace":"test-ns-5xvz7","clusterName":"fail-pbfxx"}
{"level":"info","ts":"2026-07-13T15:54:04.842Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"260e40d0-c618-40c3-bd14-1b8011cc99cc"}
{"level":"info","ts":"2026-07-13T15:54:04.842Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"260e40d0-c618-40c3-bd14-1b8011cc99cc"}
{"level":"info","ts":"2026-07-13T15:54:04.842Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"260e40d0-c618-40c3-bd14-1b8011cc99cc","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:54:04.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"260e40d0-c618-40c3-bd14-1b8011cc99cc","Found 1 head Pod":"long-running-qtcvg-head-44cnb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:54:04.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"260e40d0-c618-40c3-bd14-1b8011cc99cc","head Pod":"long-running-qtcvg-head-44cnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qtcvg-head-44cnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:54:04.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"260e40d0-c618-40c3-bd14-1b8011cc99cc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:54:04.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"260e40d0-c618-40c3-bd14-1b8011cc99cc","worker Pod":"long-running-qtcvg-small-group-worker-4jbht","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qtcvg-small-group-worker-4jbht. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:54:04.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"260e40d0-c618-40c3-bd14-1b8011cc99cc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:54:04.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"260e40d0-c618-40c3-bd14-1b8011cc99cc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:54:04.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"260e40d0-c618-40c3-bd14-1b8011cc99cc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:54:04.842Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"260e40d0-c618-40c3-bd14-1b8011cc99cc","podName":"long-running-qtcvg-head-44cnb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:53:56Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:54:04.842Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"260e40d0-c618-40c3-bd14-1b8011cc99cc","podName":"long-running-qtcvg-head-44cnb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:53:56Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:54:04.842Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"260e40d0-c618-40c3-bd14-1b8011cc99cc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:54:04.842Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"260e40d0-c618-40c3-bd14-1b8011cc99cc","seconds":300}
{"level":"info","ts":"2026-07-13T15:54:05.325Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"284e46cd-ba4c-460e-b4e9-13b4257278c3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:05.325Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"284e46cd-ba4c-460e-b4e9-13b4257278c3","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:05.325Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"284e46cd-ba4c-460e-b4e9-13b4257278c3","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:05.325Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"284e46cd-ba4c-460e-b4e9-13b4257278c3"}
{"level":"info","ts":"2026-07-13T15:54:05.325Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"284e46cd-ba4c-460e-b4e9-13b4257278c3","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:54:06.519Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-5xvz7"},"reconcileID":"20ef9c3c-4c43-47b5-bea1-b3f24700bf7f"}
{"level":"info","ts":"2026-07-13T15:54:06.531Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-5xvz7"},"reconcileID":"daf60a78-520b-4272-b8e6-2fcd78aa7f01"}
{"level":"info","ts":"2026-07-13T15:54:08.329Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d8aad830-0a46-4948-ae72-8ebf01cf88cb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:08.329Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d8aad830-0a46-4948-ae72-8ebf01cf88cb","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:08.329Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d8aad830-0a46-4948-ae72-8ebf01cf88cb","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:08.330Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d8aad830-0a46-4948-ae72-8ebf01cf88cb"}
{"level":"info","ts":"2026-07-13T15:54:08.330Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d8aad830-0a46-4948-ae72-8ebf01cf88cb","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:54:11.330Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"5ba21158-5c6c-4142-ae47-97d20e740e3d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:11.330Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"5ba21158-5c6c-4142-ae47-97d20e740e3d","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:11.330Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"5ba21158-5c6c-4142-ae47-97d20e740e3d","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:11.330Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"5ba21158-5c6c-4142-ae47-97d20e740e3d"}
{"level":"info","ts":"2026-07-13T15:54:11.330Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"5ba21158-5c6c-4142-ae47-97d20e740e3d","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:54:13.854Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4921e6e6-4867-41c7-92f1-253f02d51c86"}
{"level":"info","ts":"2026-07-13T15:54:13.854Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4921e6e6-4867-41c7-92f1-253f02d51c86"}
{"level":"info","ts":"2026-07-13T15:54:13.854Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4921e6e6-4867-41c7-92f1-253f02d51c86","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:54:13.855Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4921e6e6-4867-41c7-92f1-253f02d51c86","Found 1 head Pod":"long-running-qtcvg-head-44cnb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:54:13.855Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4921e6e6-4867-41c7-92f1-253f02d51c86","head Pod":"long-running-qtcvg-head-44cnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qtcvg-head-44cnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:54:13.855Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4921e6e6-4867-41c7-92f1-253f02d51c86","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:54:13.855Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4921e6e6-4867-41c7-92f1-253f02d51c86","worker Pod":"long-running-qtcvg-small-group-worker-4jbht","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qtcvg-small-group-worker-4jbht. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:54:13.855Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4921e6e6-4867-41c7-92f1-253f02d51c86","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:54:13.855Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4921e6e6-4867-41c7-92f1-253f02d51c86","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:54:13.855Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4921e6e6-4867-41c7-92f1-253f02d51c86","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:54:13.855Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4921e6e6-4867-41c7-92f1-253f02d51c86","podName":"long-running-qtcvg-small-group-worker-4jbht","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:54:13.855Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4921e6e6-4867-41c7-92f1-253f02d51c86","podName":"long-running-qtcvg-small-group-worker-4jbht","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:54:13.855Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4921e6e6-4867-41c7-92f1-253f02d51c86","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:54:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:54:13.855Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"4921e6e6-4867-41c7-92f1-253f02d51c86","name":"long-running-qtcvg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:54:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.22","serviceIP":"10.133.0.22","podName":"long-running-qtcvg-head-44cnb","serviceName":"long-running-qtcvg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:54:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.22","serviceIP":"10.133.0.22","podName":"long-running-qtcvg-head-44cnb","serviceName":"long-running-qtcvg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:54:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:54:13.867Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"ba5057ff-fc2f-4c6a-8ad1-e8be9f14d23d","rayCluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:13.867Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"78404012-814b-45eb-bbb6-7e8de720794e","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:13.867Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d1cf580c-6c75-4e1b-a792-6567aff3d5f1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:13.867Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"ba5057ff-fc2f-4c6a-8ad1-e8be9f14d23d","pod":"long-running-qtcvg-head-44cnb","ip":"10.133.0.22"}
{"level":"info","ts":"2026-07-13T15:54:13.867Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"ba5057ff-fc2f-4c6a-8ad1-e8be9f14d23d","pod":"long-running-qtcvg-small-group-worker-4jbht","ip":"10.134.0.28"}
{"level":"info","ts":"2026-07-13T15:54:13.867Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"ba5057ff-fc2f-4c6a-8ad1-e8be9f14d23d","rayCluster":"long-running-qtcvg","podIPs":["10.133.0.22","10.134.0.28"]}
{"level":"info","ts":"2026-07-13T15:54:13.867Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d1cf580c-6c75-4e1b-a792-6567aff3d5f1","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:13.867Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d1cf580c-6c75-4e1b-a792-6567aff3d5f1","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:13.867Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d1cf580c-6c75-4e1b-a792-6567aff3d5f1"}
{"level":"info","ts":"2026-07-13T15:54:13.867Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d1cf580c-6c75-4e1b-a792-6567aff3d5f1","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:54:13.880Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"78404012-814b-45eb-bbb6-7e8de720794e","name":"long-running-qtcvg-head"}
{"level":"info","ts":"2026-07-13T15:54:13.889Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"78404012-814b-45eb-bbb6-7e8de720794e","name":"long-running-qtcvg-workers"}
{"level":"info","ts":"2026-07-13T15:54:13.889Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"78404012-814b-45eb-bbb6-7e8de720794e","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:14.331Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a9aac6ca-ed21-4433-bd19-2bc76faf4a08","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:14.331Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a9aac6ca-ed21-4433-bd19-2bc76faf4a08","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:14.331Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a9aac6ca-ed21-4433-bd19-2bc76faf4a08","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:14.331Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a9aac6ca-ed21-4433-bd19-2bc76faf4a08"}
{"level":"info","ts":"2026-07-13T15:54:14.331Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a9aac6ca-ed21-4433-bd19-2bc76faf4a08","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:54:15.867Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9f3bfaa0-4f9c-4901-af87-85edb635a126"}
{"level":"info","ts":"2026-07-13T15:54:15.867Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9f3bfaa0-4f9c-4901-af87-85edb635a126"}
{"level":"info","ts":"2026-07-13T15:54:15.867Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9f3bfaa0-4f9c-4901-af87-85edb635a126","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:54:15.867Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9f3bfaa0-4f9c-4901-af87-85edb635a126","Found 1 head Pod":"long-running-qtcvg-head-44cnb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:54:15.867Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9f3bfaa0-4f9c-4901-af87-85edb635a126","head Pod":"long-running-qtcvg-head-44cnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qtcvg-head-44cnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:54:15.867Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9f3bfaa0-4f9c-4901-af87-85edb635a126","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:54:15.867Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9f3bfaa0-4f9c-4901-af87-85edb635a126","worker Pod":"long-running-qtcvg-small-group-worker-4jbht","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qtcvg-small-group-worker-4jbht. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:54:15.867Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9f3bfaa0-4f9c-4901-af87-85edb635a126","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:54:15.867Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9f3bfaa0-4f9c-4901-af87-85edb635a126","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:54:15.867Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9f3bfaa0-4f9c-4901-af87-85edb635a126","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:54:15.867Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9f3bfaa0-4f9c-4901-af87-85edb635a126","podName":"long-running-qtcvg-small-group-worker-4jbht","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:54:15.867Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9f3bfaa0-4f9c-4901-af87-85edb635a126","podName":"long-running-qtcvg-small-group-worker-4jbht","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:54:15.867Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9f3bfaa0-4f9c-4901-af87-85edb635a126","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:54:15.867Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"9f3bfaa0-4f9c-4901-af87-85edb635a126","seconds":300}
{"level":"info","ts":"2026-07-13T15:54:17.331Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"3bb607b7-dad4-4655-b11a-2d8abebd475d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:17.332Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"3bb607b7-dad4-4655-b11a-2d8abebd475d","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:17.332Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"3bb607b7-dad4-4655-b11a-2d8abebd475d","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:17.332Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"3bb607b7-dad4-4655-b11a-2d8abebd475d"}
{"level":"info","ts":"2026-07-13T15:54:17.332Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"3bb607b7-dad4-4655-b11a-2d8abebd475d","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:54:20.333Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c1818170-600e-4234-9019-e8ebd214f695","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:20.333Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c1818170-600e-4234-9019-e8ebd214f695","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:20.333Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c1818170-600e-4234-9019-e8ebd214f695","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:20.333Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c1818170-600e-4234-9019-e8ebd214f695"}
{"level":"info","ts":"2026-07-13T15:54:20.333Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c1818170-600e-4234-9019-e8ebd214f695","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:54:23.334Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"6db7fc33-97b1-4709-b422-10ebf48ca1eb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:23.334Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"6db7fc33-97b1-4709-b422-10ebf48ca1eb","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:23.334Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"6db7fc33-97b1-4709-b422-10ebf48ca1eb","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:23.335Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"6db7fc33-97b1-4709-b422-10ebf48ca1eb"}
{"level":"info","ts":"2026-07-13T15:54:23.335Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"6db7fc33-97b1-4709-b422-10ebf48ca1eb","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:54:23.722Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"38a01118-b361-4db4-9400-a042c4786f95","rayCluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:23.723Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"38a01118-b361-4db4-9400-a042c4786f95","pod":"long-running-qtcvg-head-44cnb","ip":"10.133.0.22"}
{"level":"info","ts":"2026-07-13T15:54:23.723Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"38a01118-b361-4db4-9400-a042c4786f95","pod":"long-running-qtcvg-small-group-worker-4jbht","ip":"10.134.0.28"}
{"level":"info","ts":"2026-07-13T15:54:23.723Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"38a01118-b361-4db4-9400-a042c4786f95","rayCluster":"long-running-qtcvg","podIPs":["10.133.0.22","10.134.0.28"]}
{"level":"info","ts":"2026-07-13T15:54:26.335Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b11bcdc1-b405-4619-8036-11b43f04a735","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:26.335Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b11bcdc1-b405-4619-8036-11b43f04a735","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:26.335Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b11bcdc1-b405-4619-8036-11b43f04a735","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:26.335Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b11bcdc1-b405-4619-8036-11b43f04a735"}
{"level":"info","ts":"2026-07-13T15:54:26.335Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b11bcdc1-b405-4619-8036-11b43f04a735","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:54:29.336Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"965f555d-273d-4100-8c6c-077afc73a37f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:29.336Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"965f555d-273d-4100-8c6c-077afc73a37f","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:29.336Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"965f555d-273d-4100-8c6c-077afc73a37f","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:29.336Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"965f555d-273d-4100-8c6c-077afc73a37f"}
{"level":"info","ts":"2026-07-13T15:54:29.336Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"965f555d-273d-4100-8c6c-077afc73a37f","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:54:32.337Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"1f76ea70-e57e-4f8f-9b30-66c36db1ace8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:32.337Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"1f76ea70-e57e-4f8f-9b30-66c36db1ace8","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:32.337Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"1f76ea70-e57e-4f8f-9b30-66c36db1ace8","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:32.337Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"1f76ea70-e57e-4f8f-9b30-66c36db1ace8"}
{"level":"info","ts":"2026-07-13T15:54:32.337Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"1f76ea70-e57e-4f8f-9b30-66c36db1ace8","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:54:34.937Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"b8a0d02f-5bad-4ad7-b09d-f5d8ae66b4bf","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:54:34.937Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"b8a0d02f-5bad-4ad7-b09d-f5d8ae66b4bf","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T15:54:34.937Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"b8a0d02f-5bad-4ad7-b09d-f5d8ae66b4bf","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T15:54:34.937Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"b8a0d02f-5bad-4ad7-b09d-f5d8ae66b4bf","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T15:54:35.338Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"fa066a09-3a30-44a8-9bbc-12e27ae40ac7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:35.338Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"fa066a09-3a30-44a8-9bbc-12e27ae40ac7","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:35.338Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"fa066a09-3a30-44a8-9bbc-12e27ae40ac7","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:35.338Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"fa066a09-3a30-44a8-9bbc-12e27ae40ac7"}
{"level":"info","ts":"2026-07-13T15:54:35.338Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"fa066a09-3a30-44a8-9bbc-12e27ae40ac7","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:54:37.793Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f8c90afd-4521-4b7e-9e0c-6d4dd6376e38"}
{"level":"info","ts":"2026-07-13T15:54:37.793Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f8c90afd-4521-4b7e-9e0c-6d4dd6376e38"}
{"level":"info","ts":"2026-07-13T15:54:37.793Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f8c90afd-4521-4b7e-9e0c-6d4dd6376e38","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:54:37.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f8c90afd-4521-4b7e-9e0c-6d4dd6376e38","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:54:37.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f8c90afd-4521-4b7e-9e0c-6d4dd6376e38","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:54:37.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f8c90afd-4521-4b7e-9e0c-6d4dd6376e38","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:54:37.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f8c90afd-4521-4b7e-9e0c-6d4dd6376e38","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:54:37.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f8c90afd-4521-4b7e-9e0c-6d4dd6376e38","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:54:37.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f8c90afd-4521-4b7e-9e0c-6d4dd6376e38","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:54:37.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f8c90afd-4521-4b7e-9e0c-6d4dd6376e38","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:54:37.793Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f8c90afd-4521-4b7e-9e0c-6d4dd6376e38","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:54:37.793Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f8c90afd-4521-4b7e-9e0c-6d4dd6376e38","seconds":300}
{"level":"info","ts":"2026-07-13T15:54:38.339Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c17dd3d4-1284-4652-958c-db12ce787811","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:38.339Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c17dd3d4-1284-4652-958c-db12ce787811","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:38.339Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c17dd3d4-1284-4652-958c-db12ce787811","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:38.339Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c17dd3d4-1284-4652-958c-db12ce787811"}
{"level":"info","ts":"2026-07-13T15:54:38.339Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c17dd3d4-1284-4652-958c-db12ce787811","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:54:41.340Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"80fe51d2-23f6-4003-9ef6-81c39928eb8c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:41.340Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"80fe51d2-23f6-4003-9ef6-81c39928eb8c","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:41.340Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"80fe51d2-23f6-4003-9ef6-81c39928eb8c","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:41.341Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"80fe51d2-23f6-4003-9ef6-81c39928eb8c"}
{"level":"info","ts":"2026-07-13T15:54:41.341Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"80fe51d2-23f6-4003-9ef6-81c39928eb8c","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:54:44.341Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"36c691ed-7d15-495d-aa91-a4be4d631158","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:44.341Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"36c691ed-7d15-495d-aa91-a4be4d631158","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:44.341Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"36c691ed-7d15-495d-aa91-a4be4d631158","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:44.341Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"36c691ed-7d15-495d-aa91-a4be4d631158"}
{"level":"info","ts":"2026-07-13T15:54:44.341Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"36c691ed-7d15-495d-aa91-a4be4d631158","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:54:47.342Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b773413d-ee86-47db-8c94-46779915a954","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:47.342Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b773413d-ee86-47db-8c94-46779915a954","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:47.342Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b773413d-ee86-47db-8c94-46779915a954","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:47.342Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b773413d-ee86-47db-8c94-46779915a954"}
{"level":"info","ts":"2026-07-13T15:54:47.342Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b773413d-ee86-47db-8c94-46779915a954","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:54:50.343Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"31034916-e412-4c97-8333-3a86d0e7edaf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:50.343Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"31034916-e412-4c97-8333-3a86d0e7edaf","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:50.343Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"31034916-e412-4c97-8333-3a86d0e7edaf","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:50.343Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"31034916-e412-4c97-8333-3a86d0e7edaf"}
{"level":"info","ts":"2026-07-13T15:54:50.343Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"31034916-e412-4c97-8333-3a86d0e7edaf","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:54:53.344Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ac6228b5-8917-4fec-8139-5abe43270c1b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:53.344Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ac6228b5-8917-4fec-8139-5abe43270c1b","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:53.344Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ac6228b5-8917-4fec-8139-5abe43270c1b","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:53.345Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ac6228b5-8917-4fec-8139-5abe43270c1b"}
{"level":"info","ts":"2026-07-13T15:54:53.345Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ac6228b5-8917-4fec-8139-5abe43270c1b","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:54:56.345Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"718862dc-6a36-4521-ad3c-78d4c2815802","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:56.345Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"718862dc-6a36-4521-ad3c-78d4c2815802","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:56.345Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"718862dc-6a36-4521-ad3c-78d4c2815802","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:56.345Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"718862dc-6a36-4521-ad3c-78d4c2815802"}
{"level":"info","ts":"2026-07-13T15:54:56.345Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"718862dc-6a36-4521-ad3c-78d4c2815802","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:54:59.345Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f8bf04cf-f99c-4241-899b-d72794fcf633","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:54:59.345Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f8bf04cf-f99c-4241-899b-d72794fcf633","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:54:59.345Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f8bf04cf-f99c-4241-899b-d72794fcf633","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:54:59.346Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f8bf04cf-f99c-4241-899b-d72794fcf633"}
{"level":"info","ts":"2026-07-13T15:54:59.346Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f8bf04cf-f99c-4241-899b-d72794fcf633","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:55:02.347Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"da003456-e37e-4242-9c7c-467934e0ec8e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:02.347Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"da003456-e37e-4242-9c7c-467934e0ec8e","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:02.347Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"da003456-e37e-4242-9c7c-467934e0ec8e","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:02.347Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"da003456-e37e-4242-9c7c-467934e0ec8e"}
{"level":"info","ts":"2026-07-13T15:55:02.347Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"da003456-e37e-4242-9c7c-467934e0ec8e","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:55:05.347Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ecb93c62-42e8-4c0a-b3a3-360fbe12007e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:05.347Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ecb93c62-42e8-4c0a-b3a3-360fbe12007e","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:05.347Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ecb93c62-42e8-4c0a-b3a3-360fbe12007e","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:05.347Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ecb93c62-42e8-4c0a-b3a3-360fbe12007e"}
{"level":"info","ts":"2026-07-13T15:55:05.347Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ecb93c62-42e8-4c0a-b3a3-360fbe12007e","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:55:08.348Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"69601ac3-aa6f-4854-a751-e74c0a65a5ff","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:08.348Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"69601ac3-aa6f-4854-a751-e74c0a65a5ff","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:08.348Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"69601ac3-aa6f-4854-a751-e74c0a65a5ff","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:08.348Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"69601ac3-aa6f-4854-a751-e74c0a65a5ff"}
{"level":"info","ts":"2026-07-13T15:55:08.348Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"69601ac3-aa6f-4854-a751-e74c0a65a5ff","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:55:11.349Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"bf29232b-57dc-4b57-9c35-8c42e922e074","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:11.349Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"bf29232b-57dc-4b57-9c35-8c42e922e074","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:11.350Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"bf29232b-57dc-4b57-9c35-8c42e922e074","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:11.350Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"bf29232b-57dc-4b57-9c35-8c42e922e074"}
{"level":"info","ts":"2026-07-13T15:55:11.350Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"bf29232b-57dc-4b57-9c35-8c42e922e074","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:55:14.351Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b3f556bb-13d9-49ff-98c3-a9dc083fbb02","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:14.351Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b3f556bb-13d9-49ff-98c3-a9dc083fbb02","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:14.351Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b3f556bb-13d9-49ff-98c3-a9dc083fbb02","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:14.351Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b3f556bb-13d9-49ff-98c3-a9dc083fbb02"}
{"level":"info","ts":"2026-07-13T15:55:14.351Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b3f556bb-13d9-49ff-98c3-a9dc083fbb02","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:55:17.280Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2edcd6aa-4ce2-4003-91fd-087f970521c8"}
{"level":"info","ts":"2026-07-13T15:55:17.280Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2edcd6aa-4ce2-4003-91fd-087f970521c8"}
{"level":"info","ts":"2026-07-13T15:55:17.280Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2edcd6aa-4ce2-4003-91fd-087f970521c8","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:17.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2edcd6aa-4ce2-4003-91fd-087f970521c8","Found 1 head Pod":"long-running-qtcvg-head-44cnb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:55:17.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2edcd6aa-4ce2-4003-91fd-087f970521c8","head Pod":"long-running-qtcvg-head-44cnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qtcvg-head-44cnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:17.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2edcd6aa-4ce2-4003-91fd-087f970521c8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:55:17.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2edcd6aa-4ce2-4003-91fd-087f970521c8","worker Pod":"long-running-qtcvg-small-group-worker-4jbht","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qtcvg-small-group-worker-4jbht. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:17.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2edcd6aa-4ce2-4003-91fd-087f970521c8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:17.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2edcd6aa-4ce2-4003-91fd-087f970521c8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:55:17.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2edcd6aa-4ce2-4003-91fd-087f970521c8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:17.280Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2edcd6aa-4ce2-4003-91fd-087f970521c8","podName":"long-running-qtcvg-small-group-worker-4jbht","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:17.280Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2edcd6aa-4ce2-4003-91fd-087f970521c8","podName":"long-running-qtcvg-small-group-worker-4jbht","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:17.280Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2edcd6aa-4ce2-4003-91fd-087f970521c8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:55:17.280Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2edcd6aa-4ce2-4003-91fd-087f970521c8","seconds":300}
{"level":"info","ts":"2026-07-13T15:55:17.351Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"cb9a3303-9b74-4e5f-a3cb-42d76008be86","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:17.351Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"cb9a3303-9b74-4e5f-a3cb-42d76008be86","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:17.351Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"cb9a3303-9b74-4e5f-a3cb-42d76008be86","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:17.352Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"cb9a3303-9b74-4e5f-a3cb-42d76008be86"}
{"level":"info","ts":"2026-07-13T15:55:17.352Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"cb9a3303-9b74-4e5f-a3cb-42d76008be86","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:55:18.285Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bcc013f8-b005-403d-a387-a1b19ce6ec9c"}
{"level":"info","ts":"2026-07-13T15:55:18.285Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bcc013f8-b005-403d-a387-a1b19ce6ec9c"}
{"level":"info","ts":"2026-07-13T15:55:18.285Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bcc013f8-b005-403d-a387-a1b19ce6ec9c","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:18.285Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bcc013f8-b005-403d-a387-a1b19ce6ec9c","Found 1 head Pod":"long-running-qtcvg-head-44cnb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:55:18.285Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bcc013f8-b005-403d-a387-a1b19ce6ec9c","head Pod":"long-running-qtcvg-head-44cnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qtcvg-head-44cnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:18.285Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bcc013f8-b005-403d-a387-a1b19ce6ec9c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:55:18.285Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bcc013f8-b005-403d-a387-a1b19ce6ec9c","worker Pod":"long-running-qtcvg-small-group-worker-4jbht","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qtcvg-small-group-worker-4jbht. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:18.285Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bcc013f8-b005-403d-a387-a1b19ce6ec9c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:18.285Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bcc013f8-b005-403d-a387-a1b19ce6ec9c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:55:18.285Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bcc013f8-b005-403d-a387-a1b19ce6ec9c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:18.285Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bcc013f8-b005-403d-a387-a1b19ce6ec9c","podName":"long-running-qtcvg-small-group-worker-4jbht","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:18.285Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bcc013f8-b005-403d-a387-a1b19ce6ec9c","podName":"long-running-qtcvg-small-group-worker-4jbht","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:18.285Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bcc013f8-b005-403d-a387-a1b19ce6ec9c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:55:18.285Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bcc013f8-b005-403d-a387-a1b19ce6ec9c","seconds":300}
{"level":"info","ts":"2026-07-13T15:55:20.352Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"de822195-8f12-4d59-ae4e-d19d5c71518b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:20.352Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"de822195-8f12-4d59-ae4e-d19d5c71518b","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:20.352Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"de822195-8f12-4d59-ae4e-d19d5c71518b","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:20.352Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"de822195-8f12-4d59-ae4e-d19d5c71518b"}
{"level":"info","ts":"2026-07-13T15:55:20.352Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"de822195-8f12-4d59-ae4e-d19d5c71518b","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:55:23.353Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"815a0661-5b76-49dd-924d-11235c6e0135","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:23.353Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"815a0661-5b76-49dd-924d-11235c6e0135","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:23.353Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"815a0661-5b76-49dd-924d-11235c6e0135","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:23.353Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"815a0661-5b76-49dd-924d-11235c6e0135"}
{"level":"info","ts":"2026-07-13T15:55:23.353Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"815a0661-5b76-49dd-924d-11235c6e0135","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:55:23.723Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"9f8678b6-69c6-412b-b5e8-5da504263b46","rayCluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:23.723Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"9f8678b6-69c6-412b-b5e8-5da504263b46","pod":"long-running-qtcvg-head-44cnb","ip":"10.133.0.22"}
{"level":"info","ts":"2026-07-13T15:55:23.723Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"9f8678b6-69c6-412b-b5e8-5da504263b46","pod":"long-running-qtcvg-small-group-worker-4jbht","ip":"10.134.0.28"}
{"level":"info","ts":"2026-07-13T15:55:23.723Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"9f8678b6-69c6-412b-b5e8-5da504263b46","rayCluster":"long-running-qtcvg","podIPs":["10.133.0.22","10.134.0.28"]}
{"level":"info","ts":"2026-07-13T15:55:26.354Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7b1697d7-8ee3-4ffc-a508-4eea387e4f47","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:26.354Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7b1697d7-8ee3-4ffc-a508-4eea387e4f47","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:26.354Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7b1697d7-8ee3-4ffc-a508-4eea387e4f47","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:26.354Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7b1697d7-8ee3-4ffc-a508-4eea387e4f47"}
{"level":"info","ts":"2026-07-13T15:55:26.354Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7b1697d7-8ee3-4ffc-a508-4eea387e4f47","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:55:28.327Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"3e33b2bc-5796-43dc-87aa-da8e7c415ebc"}
{"level":"info","ts":"2026-07-13T15:55:28.327Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"3e33b2bc-5796-43dc-87aa-da8e7c415ebc"}
{"level":"info","ts":"2026-07-13T15:55:28.327Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"3e33b2bc-5796-43dc-87aa-da8e7c415ebc","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:28.327Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"3e33b2bc-5796-43dc-87aa-da8e7c415ebc","Found 1 head Pod":"long-running-qtcvg-head-44cnb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:55:28.327Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"3e33b2bc-5796-43dc-87aa-da8e7c415ebc","head Pod":"long-running-qtcvg-head-44cnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qtcvg-head-44cnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:28.327Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"3e33b2bc-5796-43dc-87aa-da8e7c415ebc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:55:28.327Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"3e33b2bc-5796-43dc-87aa-da8e7c415ebc","worker Pod":"long-running-qtcvg-small-group-worker-4jbht","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qtcvg-small-group-worker-4jbht. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:28.327Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"3e33b2bc-5796-43dc-87aa-da8e7c415ebc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:28.327Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"3e33b2bc-5796-43dc-87aa-da8e7c415ebc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:55:28.327Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"3e33b2bc-5796-43dc-87aa-da8e7c415ebc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:28.327Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"3e33b2bc-5796-43dc-87aa-da8e7c415ebc","podName":"long-running-qtcvg-small-group-worker-4jbht","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:28.327Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"3e33b2bc-5796-43dc-87aa-da8e7c415ebc","podName":"long-running-qtcvg-small-group-worker-4jbht","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:28.327Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"3e33b2bc-5796-43dc-87aa-da8e7c415ebc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:55:28.327Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"3e33b2bc-5796-43dc-87aa-da8e7c415ebc","seconds":300}
{"level":"info","ts":"2026-07-13T15:55:29.337Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2f7d2fea-d4f1-4038-9c66-e8e5a6581bb9"}
{"level":"info","ts":"2026-07-13T15:55:29.337Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2f7d2fea-d4f1-4038-9c66-e8e5a6581bb9"}
{"level":"info","ts":"2026-07-13T15:55:29.337Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2f7d2fea-d4f1-4038-9c66-e8e5a6581bb9","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:29.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2f7d2fea-d4f1-4038-9c66-e8e5a6581bb9","Found 1 head Pod":"long-running-qtcvg-head-44cnb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:55:29.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2f7d2fea-d4f1-4038-9c66-e8e5a6581bb9","head Pod":"long-running-qtcvg-head-44cnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qtcvg-head-44cnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:29.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2f7d2fea-d4f1-4038-9c66-e8e5a6581bb9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:55:29.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2f7d2fea-d4f1-4038-9c66-e8e5a6581bb9","worker Pod":"long-running-qtcvg-small-group-worker-4jbht","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qtcvg-small-group-worker-4jbht. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:29.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2f7d2fea-d4f1-4038-9c66-e8e5a6581bb9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:29.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2f7d2fea-d4f1-4038-9c66-e8e5a6581bb9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:55:29.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2f7d2fea-d4f1-4038-9c66-e8e5a6581bb9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:29.338Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2f7d2fea-d4f1-4038-9c66-e8e5a6581bb9","podName":"long-running-qtcvg-small-group-worker-4jbht","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:53:56Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:55:29.338Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2f7d2fea-d4f1-4038-9c66-e8e5a6581bb9","podName":"long-running-qtcvg-small-group-worker-4jbht","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:53:56Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:55:29.338Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2f7d2fea-d4f1-4038-9c66-e8e5a6581bb9","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:55:29.338Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"2f7d2fea-d4f1-4038-9c66-e8e5a6581bb9","name":"long-running-qtcvg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:54:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.22","serviceIP":"10.133.0.22","podName":"long-running-qtcvg-head-44cnb","serviceName":"long-running-qtcvg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:54:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:55:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.22","serviceIP":"10.133.0.22","podName":"long-running-qtcvg-head-44cnb","serviceName":"long-running-qtcvg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:54:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:55:29.350Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"8acf845f-a1d9-42b4-a206-bb9f07083fd8","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:29.350Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"b1093591-06e4-4512-962e-68a2e3671fb6","rayCluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:29.350Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"06eb2f2b-d365-4b78-b9c2-7cbab9dc5e3d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:29.350Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"06eb2f2b-d365-4b78-b9c2-7cbab9dc5e3d","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:29.350Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"b1093591-06e4-4512-962e-68a2e3671fb6","pod":"long-running-qtcvg-small-group-worker-4jbht","ip":"10.134.0.28"}
{"level":"info","ts":"2026-07-13T15:55:29.350Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"06eb2f2b-d365-4b78-b9c2-7cbab9dc5e3d","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:29.350Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"b1093591-06e4-4512-962e-68a2e3671fb6","pod":"long-running-qtcvg-head-44cnb","ip":"10.133.0.22"}
{"level":"info","ts":"2026-07-13T15:55:29.350Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"06eb2f2b-d365-4b78-b9c2-7cbab9dc5e3d"}
{"level":"info","ts":"2026-07-13T15:55:29.350Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"06eb2f2b-d365-4b78-b9c2-7cbab9dc5e3d","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:55:29.350Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"b1093591-06e4-4512-962e-68a2e3671fb6","rayCluster":"long-running-qtcvg","podIPs":["10.134.0.28","10.133.0.22"]}
{"level":"info","ts":"2026-07-13T15:55:29.354Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a35d013a-ebfe-4399-a551-0787630ac02d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:29.354Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a35d013a-ebfe-4399-a551-0787630ac02d","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:29.354Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a35d013a-ebfe-4399-a551-0787630ac02d","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:29.355Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a35d013a-ebfe-4399-a551-0787630ac02d"}
{"level":"info","ts":"2026-07-13T15:55:29.355Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a35d013a-ebfe-4399-a551-0787630ac02d","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:55:29.364Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"8acf845f-a1d9-42b4-a206-bb9f07083fd8","name":"long-running-qtcvg-head"}
{"level":"info","ts":"2026-07-13T15:55:29.375Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"8acf845f-a1d9-42b4-a206-bb9f07083fd8","name":"long-running-qtcvg-workers"}
{"level":"info","ts":"2026-07-13T15:55:29.375Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"8acf845f-a1d9-42b4-a206-bb9f07083fd8","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:31.350Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0ed3aac7-3c1d-4ff0-9fef-9be1b0b5a1a9"}
{"level":"info","ts":"2026-07-13T15:55:31.350Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0ed3aac7-3c1d-4ff0-9fef-9be1b0b5a1a9"}
{"level":"info","ts":"2026-07-13T15:55:31.351Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0ed3aac7-3c1d-4ff0-9fef-9be1b0b5a1a9","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:31.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0ed3aac7-3c1d-4ff0-9fef-9be1b0b5a1a9","Found 1 head Pod":"long-running-qtcvg-head-44cnb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:55:31.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0ed3aac7-3c1d-4ff0-9fef-9be1b0b5a1a9","head Pod":"long-running-qtcvg-head-44cnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qtcvg-head-44cnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:31.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0ed3aac7-3c1d-4ff0-9fef-9be1b0b5a1a9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:55:31.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0ed3aac7-3c1d-4ff0-9fef-9be1b0b5a1a9","worker Pod":"long-running-qtcvg-small-group-worker-4jbht","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qtcvg-small-group-worker-4jbht. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:31.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0ed3aac7-3c1d-4ff0-9fef-9be1b0b5a1a9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:31.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0ed3aac7-3c1d-4ff0-9fef-9be1b0b5a1a9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:55:31.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0ed3aac7-3c1d-4ff0-9fef-9be1b0b5a1a9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:31.351Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0ed3aac7-3c1d-4ff0-9fef-9be1b0b5a1a9","podName":"long-running-qtcvg-small-group-worker-4jbht","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:53:56Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:55:31.351Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0ed3aac7-3c1d-4ff0-9fef-9be1b0b5a1a9","podName":"long-running-qtcvg-small-group-worker-4jbht","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:53:56Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:55:31.351Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0ed3aac7-3c1d-4ff0-9fef-9be1b0b5a1a9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:55:31.351Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"0ed3aac7-3c1d-4ff0-9fef-9be1b0b5a1a9","seconds":300}
{"level":"info","ts":"2026-07-13T15:55:32.355Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"1bf12beb-c85a-481a-b983-3e295742ee58","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:32.355Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"1bf12beb-c85a-481a-b983-3e295742ee58","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:32.355Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"1bf12beb-c85a-481a-b983-3e295742ee58","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:32.355Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"1bf12beb-c85a-481a-b983-3e295742ee58"}
{"level":"info","ts":"2026-07-13T15:55:32.355Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"1bf12beb-c85a-481a-b983-3e295742ee58","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:55:34.938Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8e74fa75-bbd3-464c-9655-5ef80419fef4","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:55:34.938Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8e74fa75-bbd3-464c-9655-5ef80419fef4","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T15:55:34.938Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8e74fa75-bbd3-464c-9655-5ef80419fef4","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T15:55:34.938Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8e74fa75-bbd3-464c-9655-5ef80419fef4","rayCluster":"raycluster","podIPs":["10.133.0.18","10.134.0.25"]}
{"level":"info","ts":"2026-07-13T15:55:35.355Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ec2a6474-526e-439b-8c52-787d8062173c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:35.355Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ec2a6474-526e-439b-8c52-787d8062173c","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:35.355Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ec2a6474-526e-439b-8c52-787d8062173c","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:35.356Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ec2a6474-526e-439b-8c52-787d8062173c"}
{"level":"info","ts":"2026-07-13T15:55:35.356Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ec2a6474-526e-439b-8c52-787d8062173c","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:55:38.356Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"23c096a5-11db-4a00-b84f-3c645e14475c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:38.356Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"23c096a5-11db-4a00-b84f-3c645e14475c","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:38.356Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"23c096a5-11db-4a00-b84f-3c645e14475c","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:38.356Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"23c096a5-11db-4a00-b84f-3c645e14475c"}
{"level":"info","ts":"2026-07-13T15:55:38.356Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"23c096a5-11db-4a00-b84f-3c645e14475c","RayCluster":"long-running-qtcvg","State":""}
{"level":"info","ts":"2026-07-13T15:55:40.360Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"00667ce0-422e-4187-8e4d-6cd366920df1"}
{"level":"info","ts":"2026-07-13T15:55:40.360Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"00667ce0-422e-4187-8e4d-6cd366920df1"}
{"level":"info","ts":"2026-07-13T15:55:40.360Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"00667ce0-422e-4187-8e4d-6cd366920df1","1 head service found":"long-running-qtcvg-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:40.360Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"00667ce0-422e-4187-8e4d-6cd366920df1","Found 1 head Pod":"long-running-qtcvg-head-44cnb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:55:40.360Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"00667ce0-422e-4187-8e4d-6cd366920df1","head Pod":"long-running-qtcvg-head-44cnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qtcvg-head-44cnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:40.360Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"00667ce0-422e-4187-8e4d-6cd366920df1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:55:40.360Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"00667ce0-422e-4187-8e4d-6cd366920df1","worker Pod":"long-running-qtcvg-small-group-worker-4jbht","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qtcvg-small-group-worker-4jbht. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:40.360Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"00667ce0-422e-4187-8e4d-6cd366920df1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:40.360Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"00667ce0-422e-4187-8e4d-6cd366920df1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:55:40.360Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"00667ce0-422e-4187-8e4d-6cd366920df1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:40.360Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"00667ce0-422e-4187-8e4d-6cd366920df1","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-13T15:55:40.360Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"00667ce0-422e-4187-8e4d-6cd366920df1","name":"long-running-qtcvg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:55:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.22","serviceIP":"10.133.0.22","podName":"long-running-qtcvg-head-44cnb","serviceName":"long-running-qtcvg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:54:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:55:40Z","stateTransitionTimes":{"ready":"2026-07-13T15:55:40Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.22","serviceIP":"10.133.0.22","podName":"long-running-qtcvg-head-44cnb","serviceName":"long-running-qtcvg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:54:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:55:40Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:53:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:53:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:55:40.373Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"c65bcf4f-4e44-4ec5-8a37-ca2d176a28e7","rayCluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.373Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"86189202-2c64-4a90-83e2-fafc2d298636","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:40.373Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"86189202-2c64-4a90-83e2-fafc2d298636","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.373Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"86189202-2c64-4a90-83e2-fafc2d298636","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:40.373Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"d6e86a32-48fc-4289-b448-e05aed674ce8","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.373Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"c65bcf4f-4e44-4ec5-8a37-ca2d176a28e7","pod":"long-running-qtcvg-head-44cnb","ip":"10.133.0.22"}
{"level":"info","ts":"2026-07-13T15:55:40.373Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"c65bcf4f-4e44-4ec5-8a37-ca2d176a28e7","pod":"long-running-qtcvg-small-group-worker-4jbht","ip":"10.134.0.28"}
{"level":"info","ts":"2026-07-13T15:55:40.373Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"c65bcf4f-4e44-4ec5-8a37-ca2d176a28e7","rayCluster":"long-running-qtcvg","podIPs":["10.133.0.22","10.134.0.28"]}
{"level":"info","ts":"2026-07-13T15:55:40.374Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"86189202-2c64-4a90-83e2-fafc2d298636"}
{"level":"info","ts":"2026-07-13T15:55:40.374Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"86189202-2c64-4a90-83e2-fafc2d298636","head service name":"long-running-qtcvg-head-svc","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:55:40.374Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"86189202-2c64-4a90-83e2-fafc2d298636","head service URL":"long-running-qtcvg-head-svc.test-ns-88gnn.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-13T15:55:40.374Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"86189202-2c64-4a90-83e2-fafc2d298636"}
{"level":"info","ts":"2026-07-13T15:55:40.374Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"86189202-2c64-4a90-83e2-fafc2d298636","command":["if","!","ray","job","status","--address","http://long-running-qtcvg-head-svc.test-ns-88gnn.svc.cluster.local:8265","long-running-wmk2z",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://long-running-qtcvg-head-svc.test-ns-88gnn.svc.cluster.local:8265","--no-wait","--submission-id","long-running-wmk2z","--","python /home/ray/jobs/long_running.py",";","fi",";","ray","job","logs","--address","http://long-running-qtcvg-head-svc.test-ns-88gnn.svc.cluster.local:8265","--follow","long-running-wmk2z"]}
{"level":"info","ts":"2026-07-13T15:55:40.386Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"86189202-2c64-4a90-83e2-fafc2d298636","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-07-13T15:55:40.386Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"86189202-2c64-4a90-83e2-fafc2d298636","SubmissionMode":"K8sJobMode","RayCluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.386Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"86189202-2c64-4a90-83e2-fafc2d298636","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-wmk2z","rayClusterName":"long-running-qtcvg","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:53:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-wmk2z","rayClusterName":"long-running-qtcvg","dashboardURL":"long-running-qtcvg-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T15:53:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:55:40.387Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"86189202-2c64-4a90-83e2-fafc2d298636","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T15:55:40.390Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"d6e86a32-48fc-4289-b448-e05aed674ce8","name":"long-running-qtcvg-head"}
{"level":"info","ts":"2026-07-13T15:55:40.397Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"479c66a4-ed67-4f29-830d-70172940dc9f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:40.398Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"479c66a4-ed67-4f29-830d-70172940dc9f","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.398Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"479c66a4-ed67-4f29-830d-70172940dc9f","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:40.398Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"479c66a4-ed67-4f29-830d-70172940dc9f"}
{"level":"info","ts":"2026-07-13T15:55:40.403Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"d6e86a32-48fc-4289-b448-e05aed674ce8","name":"long-running-qtcvg-workers"}
{"level":"info","ts":"2026-07-13T15:55:40.403Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"d6e86a32-48fc-4289-b448-e05aed674ce8","cluster":"long-running-qtcvg"}
{"level":"error","ts":"2026-07-13T15:55:40.421Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"479c66a4-ed67-4f29-830d-70172940dc9f","JobId":"long-running-wmk2z","error":"Job long-running-wmk2z does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:40.421Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"479c66a4-ed67-4f29-830d-70172940dc9f"}
{"level":"error","ts":"2026-07-13T15:55:40.421Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"479c66a4-ed67-4f29-830d-70172940dc9f","error":"Job long-running-wmk2z does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:40.421Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9abddbd0-12e3-46c8-9fa1-821b7cfa589d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:40.421Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9abddbd0-12e3-46c8-9fa1-821b7cfa589d","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.421Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9abddbd0-12e3-46c8-9fa1-821b7cfa589d","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:40.422Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9abddbd0-12e3-46c8-9fa1-821b7cfa589d"}
{"level":"error","ts":"2026-07-13T15:55:40.427Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9abddbd0-12e3-46c8-9fa1-821b7cfa589d","JobId":"long-running-wmk2z","error":"Job long-running-wmk2z does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:40.427Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9abddbd0-12e3-46c8-9fa1-821b7cfa589d"}
{"level":"error","ts":"2026-07-13T15:55:40.427Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9abddbd0-12e3-46c8-9fa1-821b7cfa589d","error":"Job long-running-wmk2z does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:40.427Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"21b29b6a-2b03-4b91-a517-7af3b75e8da7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:40.427Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"21b29b6a-2b03-4b91-a517-7af3b75e8da7","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.427Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"21b29b6a-2b03-4b91-a517-7af3b75e8da7","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:40.427Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"21b29b6a-2b03-4b91-a517-7af3b75e8da7"}
{"level":"error","ts":"2026-07-13T15:55:40.431Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"21b29b6a-2b03-4b91-a517-7af3b75e8da7","JobId":"long-running-wmk2z","error":"Job long-running-wmk2z does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:40.431Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"21b29b6a-2b03-4b91-a517-7af3b75e8da7"}
{"level":"error","ts":"2026-07-13T15:55:40.431Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"21b29b6a-2b03-4b91-a517-7af3b75e8da7","error":"Job long-running-wmk2z does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:40.437Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"6959cb79-d4e8-408d-8c09-4bfc9a57f2f0","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:40.437Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"6959cb79-d4e8-408d-8c09-4bfc9a57f2f0","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.437Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"6959cb79-d4e8-408d-8c09-4bfc9a57f2f0","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:40.437Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"6959cb79-d4e8-408d-8c09-4bfc9a57f2f0"}
{"level":"error","ts":"2026-07-13T15:55:40.441Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"6959cb79-d4e8-408d-8c09-4bfc9a57f2f0","JobId":"long-running-wmk2z","error":"Job long-running-wmk2z does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:40.441Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"6959cb79-d4e8-408d-8c09-4bfc9a57f2f0"}
{"level":"error","ts":"2026-07-13T15:55:40.441Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"6959cb79-d4e8-408d-8c09-4bfc9a57f2f0","error":"Job long-running-wmk2z does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:40.481Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"48752443-511b-4811-82ef-3e711dbf45c6","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:40.481Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"48752443-511b-4811-82ef-3e711dbf45c6","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.481Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"48752443-511b-4811-82ef-3e711dbf45c6","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:40.481Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"48752443-511b-4811-82ef-3e711dbf45c6"}
{"level":"error","ts":"2026-07-13T15:55:40.485Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"48752443-511b-4811-82ef-3e711dbf45c6","JobId":"long-running-wmk2z","error":"Job long-running-wmk2z does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:40.485Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"48752443-511b-4811-82ef-3e711dbf45c6"}
{"level":"error","ts":"2026-07-13T15:55:40.485Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"48752443-511b-4811-82ef-3e711dbf45c6","error":"Job long-running-wmk2z does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:40.566Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"667f3053-2b6c-4b81-963b-7ccfc7f1a764","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:40.566Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"667f3053-2b6c-4b81-963b-7ccfc7f1a764","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.566Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"667f3053-2b6c-4b81-963b-7ccfc7f1a764","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:40.566Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"667f3053-2b6c-4b81-963b-7ccfc7f1a764"}
{"level":"error","ts":"2026-07-13T15:55:40.570Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"667f3053-2b6c-4b81-963b-7ccfc7f1a764","JobId":"long-running-wmk2z","error":"Job long-running-wmk2z does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:40.570Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"667f3053-2b6c-4b81-963b-7ccfc7f1a764"}
{"level":"error","ts":"2026-07-13T15:55:40.570Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"667f3053-2b6c-4b81-963b-7ccfc7f1a764","error":"Job long-running-wmk2z does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:40.731Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"51c2cef8-958a-4d23-aafe-6c62ecf5cdaf","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:40.731Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"51c2cef8-958a-4d23-aafe-6c62ecf5cdaf","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.731Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"51c2cef8-958a-4d23-aafe-6c62ecf5cdaf","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:40.731Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"51c2cef8-958a-4d23-aafe-6c62ecf5cdaf"}
{"level":"error","ts":"2026-07-13T15:55:40.736Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"51c2cef8-958a-4d23-aafe-6c62ecf5cdaf","JobId":"long-running-wmk2z","error":"Job long-running-wmk2z does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:40.736Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"51c2cef8-958a-4d23-aafe-6c62ecf5cdaf"}
{"level":"error","ts":"2026-07-13T15:55:40.736Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"51c2cef8-958a-4d23-aafe-6c62ecf5cdaf","error":"Job long-running-wmk2z does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:40.782Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d7168022-3f17-4c51-8c3b-8cd7d9cdf5da","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:40.782Z","logger":"controllers.RayJob","msg":"Try to transition the status to `Suspending`","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d7168022-3f17-4c51-8c3b-8cd7d9cdf5da","oldStatus":"Running"}
{"level":"info","ts":"2026-07-13T15:55:40.782Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d7168022-3f17-4c51-8c3b-8cd7d9cdf5da","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-wmk2z","rayClusterName":"long-running-qtcvg","dashboardURL":"long-running-qtcvg-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T15:53:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-wmk2z","rayClusterName":"long-running-qtcvg","dashboardURL":"long-running-qtcvg-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobDeploymentStatus":"Suspending","startTime":"2026-07-13T15:53:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:55:40.782Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d7168022-3f17-4c51-8c3b-8cd7d9cdf5da","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Suspending"}
{"level":"info","ts":"2026-07-13T15:55:40.793Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"eb3414c5-0048-4641-8b4f-b5049e799a61","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:40.802Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"eb3414c5-0048-4641-8b4f-b5049e799a61","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:40.802Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"eb3414c5-0048-4641-8b4f-b5049e799a61","isClusterDeleted":false}
{"level":"info","ts":"2026-07-13T15:55:40.803Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"a2210769-1824-408d-b3b3-bfc5401fad29","namespacedName":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:40.803Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"a2210769-1824-408d-b3b3-bfc5401fad29","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.803Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"a2210769-1824-408d-b3b3-bfc5401fad29","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.803Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"f197c209-bdcf-493a-b0ec-15ba16c4ffbf"}
{"level":"info","ts":"2026-07-13T15:55:40.803Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"a2210769-1824-408d-b3b3-bfc5401fad29","namespace":"test-ns-88gnn","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-13T15:55:40.803Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"fd4b51f9-4cf7-4ab6-aea7-22c192a68f0b","rayCluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.803Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"reconcileID":"bfc662d7-c960-49d4-9844-db6d728415d0"}
{"level":"info","ts":"2026-07-13T15:55:40.803Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"fd4b51f9-4cf7-4ab6-aea7-22c192a68f0b","pod":"long-running-qtcvg-head-44cnb","ip":"10.133.0.22"}
{"level":"info","ts":"2026-07-13T15:55:40.803Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"fd4b51f9-4cf7-4ab6-aea7-22c192a68f0b","pod":"long-running-qtcvg-small-group-worker-4jbht","ip":"10.134.0.28"}
{"level":"info","ts":"2026-07-13T15:55:40.803Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"fd4b51f9-4cf7-4ab6-aea7-22c192a68f0b","rayCluster":"long-running-qtcvg","podIPs":["10.133.0.22","10.134.0.28"]}
{"level":"info","ts":"2026-07-13T15:55:40.806Z","logger":"controllers.RayJob","msg":"The associated submitter Kubernetes Job for RayJob is deleted","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"eb3414c5-0048-4641-8b4f-b5049e799a61","Submitter K8s Job":"long-running"}
{"level":"info","ts":"2026-07-13T15:55:40.807Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"eb3414c5-0048-4641-8b4f-b5049e799a61","isJobDeleted":false}
{"level":"info","ts":"2026-07-13T15:55:40.807Z","logger":"controllers.RayJob","msg":"The release of the compute resources has not been completed yet. Wait for the resources to be deleted before the status transitions to avoid a resource leak.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"eb3414c5-0048-4641-8b4f-b5049e799a61"}
{"level":"info","ts":"2026-07-13T15:55:40.807Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"67470715-7153-455b-a6cc-2e8b5ece5470","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:40.807Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"67470715-7153-455b-a6cc-2e8b5ece5470","RayCluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.807Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"67470715-7153-455b-a6cc-2e8b5ece5470","isClusterDeleted":false}
{"level":"info","ts":"2026-07-13T15:55:40.807Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"67470715-7153-455b-a6cc-2e8b5ece5470","Kubernetes Job":""}
{"level":"info","ts":"2026-07-13T15:55:40.807Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"67470715-7153-455b-a6cc-2e8b5ece5470","isJobDeleted":true}
{"level":"info","ts":"2026-07-13T15:55:40.807Z","logger":"controllers.RayJob","msg":"The release of the compute resources has not been completed yet. Wait for the resources to be deleted before the status transitions to avoid a resource leak.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"67470715-7153-455b-a6cc-2e8b5ece5470"}
{"level":"info","ts":"2026-07-13T15:55:40.807Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"863ac5e7-5912-4167-989a-50cb5d105c3a","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:40.807Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"863ac5e7-5912-4167-989a-50cb5d105c3a","RayCluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.807Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"863ac5e7-5912-4167-989a-50cb5d105c3a","isClusterDeleted":false}
{"level":"info","ts":"2026-07-13T15:55:40.807Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"863ac5e7-5912-4167-989a-50cb5d105c3a","Kubernetes Job":""}
{"level":"info","ts":"2026-07-13T15:55:40.807Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"863ac5e7-5912-4167-989a-50cb5d105c3a","isJobDeleted":true}
{"level":"info","ts":"2026-07-13T15:55:40.807Z","logger":"controllers.RayJob","msg":"The release of the compute resources has not been completed yet. Wait for the resources to be deleted before the status transitions to avoid a resource leak.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"863ac5e7-5912-4167-989a-50cb5d105c3a"}
{"level":"info","ts":"2026-07-13T15:55:40.810Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"a2210769-1824-408d-b3b3-bfc5401fad29","name":"kuberay-gateway-access","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:55:40.820Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"a2210769-1824-408d-b3b3-bfc5401fad29","name":"test-ns-88gnn-long-running-qtcvg","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-13T15:55:40.826Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"a2210769-1824-408d-b3b3-bfc5401fad29","configMap":"kube-rbac-proxy-config-long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.830Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"a2210769-1824-408d-b3b3-bfc5401fad29","serviceAccount":"long-running-qtcvg-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:55:40.848Z","logger":"raycluster-resource","msg":"default","name":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.848Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-qtcvg","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:55:40.857Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"a2210769-1824-408d-b3b3-bfc5401fad29","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.857Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"a9a63b82-5aea-4200-ba9f-b4061abef524","namespacedName":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:40.857Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"a9a63b82-5aea-4200-ba9f-b4061abef524","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.857Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"a9a63b82-5aea-4200-ba9f-b4061abef524","cluster":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.857Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"a9a63b82-5aea-4200-ba9f-b4061abef524","namespace":"test-ns-88gnn","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-13T15:55:40.857Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"a9a63b82-5aea-4200-ba9f-b4061abef524","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-13T15:55:40.858Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"ecba6a31-2acb-4a64-bb16-a211ee0d5e43","namespace":"test-ns-88gnn","clusterName":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.858Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"a260110b-c6d7-45d9-b3cf-0409e45b598c"}
{"level":"info","ts":"2026-07-13T15:55:40.858Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"aa82dbd4-3b14-4192-80fd-ff2277994a24","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:40.858Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"aa82dbd4-3b14-4192-80fd-ff2277994a24","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:40.858Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"aa82dbd4-3b14-4192-80fd-ff2277994a24","isClusterDeleted":true}
{"level":"info","ts":"2026-07-13T15:55:40.858Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"aa82dbd4-3b14-4192-80fd-ff2277994a24","Kubernetes Job":""}
{"level":"info","ts":"2026-07-13T15:55:40.858Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"aa82dbd4-3b14-4192-80fd-ff2277994a24","isJobDeleted":true}
{"level":"info","ts":"2026-07-13T15:55:40.858Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"aa82dbd4-3b14-4192-80fd-ff2277994a24","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-wmk2z","rayClusterName":"long-running-qtcvg","dashboardURL":"long-running-qtcvg-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobDeploymentStatus":"Suspending","startTime":"2026-07-13T15:53:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-07-13T15:53:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:55:40.858Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"aa82dbd4-3b14-4192-80fd-ff2277994a24","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspending","new JobDeploymentStatus":"Suspended"}
{"level":"info","ts":"2026-07-13T15:55:40.922Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"da3bf1b6-926b-4fa9-8af6-a7a62b981cf5"}
{"level":"error","ts":"2026-07-13T15:55:40.922Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"a9a63b82-5aea-4200-ba9f-b4061abef524","error":"rayclusters.ray.io \"long-running-qtcvg\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:40.922Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"a9a63b82-5aea-4200-ba9f-b4061abef524"}
{"level":"error","ts":"2026-07-13T15:55:40.922Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"a9a63b82-5aea-4200-ba9f-b4061abef524","error":"rayclusters.ray.io \"long-running-qtcvg\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:40.922Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"f1e9aaa5-0af2-4a0c-b9fb-23016f72c952","namespacedName":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:40.922Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"f1e9aaa5-0af2-4a0c-b9fb-23016f72c952"}
{"level":"info","ts":"2026-07-13T15:55:40.922Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"f1e9aaa5-0af2-4a0c-b9fb-23016f72c952","namespace":"test-ns-88gnn","authClusters":0}
{"level":"info","ts":"2026-07-13T15:55:40.922Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"f1e9aaa5-0af2-4a0c-b9fb-23016f72c952","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:55:40.922Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"40c31c47-0b1f-49db-9ab5-341c9a0abccd","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:40.927Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"98928c7a-4c96-49b3-9f35-0398a3f8c8e3","namespacedName":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:40.927Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"98928c7a-4c96-49b3-9f35-0398a3f8c8e3"}
{"level":"error","ts":"2026-07-13T15:55:40.927Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"ecba6a31-2acb-4a64-bb16-a211ee0d5e43","issuer":"ray-selfsigned-issuer-long-running-qtcvg","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qtcvg\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:40.927Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"98928c7a-4c96-49b3-9f35-0398a3f8c8e3","namespace":"test-ns-88gnn","authClusters":0}
{"level":"info","ts":"2026-07-13T15:55:40.927Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"98928c7a-4c96-49b3-9f35-0398a3f8c8e3","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:55:40.944Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"ecba6a31-2acb-4a64-bb16-a211ee0d5e43","errorCount":1}
{"level":"info","ts":"2026-07-13T15:55:40.944Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"ecba6a31-2acb-4a64-bb16-a211ee0d5e43"}
{"level":"error","ts":"2026-07-13T15:55:40.944Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"ecba6a31-2acb-4a64-bb16-a211ee0d5e43","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qtcvg\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:40.950Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"5d0b0175-8a64-4ccc-b54d-5bda7c09e5d6","namespace":"test-ns-88gnn","clusterName":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:40.950Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qtcvg","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qtcvg","reconcileID":"5d0b0175-8a64-4ccc-b54d-5bda7c09e5d6","namespace":"test-ns-88gnn","clusterName":"long-running-qtcvg"}
{"level":"info","ts":"2026-07-13T15:55:41.057Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f2aaffc6-fb7e-454c-95e7-9889292c0281","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:41.808Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c15e06ee-7eba-4b2e-87ed-ddaccafb97c4","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:41.808Z","logger":"controllers.RayJob","msg":"The status is 'Suspended', but the suspend flag is false. Transition the status to 'New'.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c15e06ee-7eba-4b2e-87ed-ddaccafb97c4"}
{"level":"info","ts":"2026-07-13T15:55:41.808Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c15e06ee-7eba-4b2e-87ed-ddaccafb97c4","oldRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-07-13T15:53:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"startTime":"2026-07-13T15:53:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:55:41.809Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c15e06ee-7eba-4b2e-87ed-ddaccafb97c4","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspended","new JobDeploymentStatus":""}
{"level":"info","ts":"2026-07-13T15:55:41.818Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b604d967-81f5-42c9-93d8-7a84926c0bf4","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:41.818Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b604d967-81f5-42c9-93d8-7a84926c0bf4"}
{"level":"info","ts":"2026-07-13T15:55:41.818Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b604d967-81f5-42c9-93d8-7a84926c0bf4","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-13T15:55:41.818Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b604d967-81f5-42c9-93d8-7a84926c0bf4","oldRayJobStatus":{"rayJobInfo":{},"startTime":"2026-07-13T15:53:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-tsqx8","rayClusterName":"long-running-qlk27","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:55:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:55:41.818Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b604d967-81f5-42c9-93d8-7a84926c0bf4","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T15:55:41.828Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"28438c70-545f-47c6-a71f-1c59a3ee3b2f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:41.828Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"28438c70-545f-47c6-a71f-1c59a3ee3b2f","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:41.828Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"28438c70-545f-47c6-a71f-1c59a3ee3b2f","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:41.828Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"28438c70-545f-47c6-a71f-1c59a3ee3b2f","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:41.832Z","logger":"raycluster-resource","msg":"default","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:41.832Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-qlk27","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:55:41.840Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"28438c70-545f-47c6-a71f-1c59a3ee3b2f","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:41.840Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"f8b5bdc3-f49e-44d3-beee-708828114abf","rayCluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:41.840Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"f8b5bdc3-f49e-44d3-beee-708828114abf","rayCluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:41.840Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a13b5229-a05e-445c-8b28-3d8af6f98250","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:41.840Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"28438c70-545f-47c6-a71f-1c59a3ee3b2f"}
{"level":"info","ts":"2026-07-13T15:55:41.840Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"199afe0e-9651-4713-aade-1ffff60d67a1","namespacedName":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:41.841Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"28438c70-545f-47c6-a71f-1c59a3ee3b2f","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:55:41.841Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"199afe0e-9651-4713-aade-1ffff60d67a1","mode":"IntegratedOAuth","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:41.840Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"7678c389-1cd2-4e6e-a33a-21b08fc51627"}
{"level":"info","ts":"2026-07-13T15:55:41.841Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"7678c389-1cd2-4e6e-a33a-21b08fc51627"}
{"level":"info","ts":"2026-07-13T15:55:41.841Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"199afe0e-9651-4713-aade-1ffff60d67a1","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:41.841Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"199afe0e-9651-4713-aade-1ffff60d67a1","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:41.841Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"45708617-3bbf-4a97-969d-7eed44471273","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:41.841Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"45708617-3bbf-4a97-969d-7eed44471273","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:41.841Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"45708617-3bbf-4a97-969d-7eed44471273","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:41.841Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"45708617-3bbf-4a97-969d-7eed44471273"}
{"level":"info","ts":"2026-07-13T15:55:41.841Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"45708617-3bbf-4a97-969d-7eed44471273","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:55:41.846Z","logger":"raycluster-resource","msg":"default","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:41.846Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-qlk27","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:55:41.849Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"7678c389-1cd2-4e6e-a33a-21b08fc51627","name":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:41.849Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"7678c389-1cd2-4e6e-a33a-21b08fc51627","error":"head certificate secret ray-head-secret-long-running-qlk27 not found"}
{"level":"error","ts":"2026-07-13T15:55:41.849Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"7678c389-1cd2-4e6e-a33a-21b08fc51627","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:41.921Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"7678c389-1cd2-4e6e-a33a-21b08fc51627","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found"}
{"level":"info","ts":"2026-07-13T15:55:41.921Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"7678c389-1cd2-4e6e-a33a-21b08fc51627","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:41.921Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"0e903461-00af-4d48-a91f-3d393d5369db","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:41.921Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"0e903461-00af-4d48-a91f-3d393d5369db","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:41.921Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"0e903461-00af-4d48-a91f-3d393d5369db","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:41.921Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a13b5229-a05e-445c-8b28-3d8af6f98250","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:41.921Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"cc32cca4-71c4-4f36-999d-14655118291a","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:41.921Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"0e903461-00af-4d48-a91f-3d393d5369db"}
{"level":"info","ts":"2026-07-13T15:55:41.921Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"0e903461-00af-4d48-a91f-3d393d5369db","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:55:41.921Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"7678c389-1cd2-4e6e-a33a-21b08fc51627","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:41.921Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"7678c389-1cd2-4e6e-a33a-21b08fc51627","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:41.921Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"7678c389-1cd2-4e6e-a33a-21b08fc51627","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:55:41.921Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"7678c389-1cd2-4e6e-a33a-21b08fc51627","name":"long-running-qlk27","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:55:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-qlk27-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:55:41.922Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"199afe0e-9651-4713-aade-1ffff60d67a1","name":"long-running-qlk27-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:55:41.922Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"f8b5bdc3-f49e-44d3-beee-708828114abf","rayCluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:41.922Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"f8b5bdc3-f49e-44d3-beee-708828114abf","rayCluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:41.927Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"199afe0e-9651-4713-aade-1ffff60d67a1","name":"kuberay-gateway-access","namespace":"test-ns-88gnn","operation":"created"}
{"level":"info","ts":"2026-07-13T15:55:41.927Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"7678c389-1cd2-4e6e-a33a-21b08fc51627","name":"long-running-qlk27","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qlk27\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-88gnn","name":"long-running-qlk27"}}
{"level":"info","ts":"2026-07-13T15:55:41.927Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"7678c389-1cd2-4e6e-a33a-21b08fc51627"}
{"level":"error","ts":"2026-07-13T15:55:41.927Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"7678c389-1cd2-4e6e-a33a-21b08fc51627","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:41.928Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"bc4d6513-f932-441c-bf0c-df8cf6307f6e"}
{"level":"info","ts":"2026-07-13T15:55:41.928Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"bc4d6513-f932-441c-bf0c-df8cf6307f6e"}
{"level":"info","ts":"2026-07-13T15:55:41.928Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"bc4d6513-f932-441c-bf0c-df8cf6307f6e","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:41.928Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"bc4d6513-f932-441c-bf0c-df8cf6307f6e","error":"head certificate secret ray-head-secret-long-running-qlk27 not found"}
{"level":"error","ts":"2026-07-13T15:55:41.928Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"bc4d6513-f932-441c-bf0c-df8cf6307f6e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:41.928Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"bc4d6513-f932-441c-bf0c-df8cf6307f6e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found"}
{"level":"info","ts":"2026-07-13T15:55:41.928Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"bc4d6513-f932-441c-bf0c-df8cf6307f6e","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:41.928Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"bc4d6513-f932-441c-bf0c-df8cf6307f6e","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:41.928Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"bc4d6513-f932-441c-bf0c-df8cf6307f6e","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:41.928Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"bc4d6513-f932-441c-bf0c-df8cf6307f6e","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:55:41.928Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"bc4d6513-f932-441c-bf0c-df8cf6307f6e","name":"long-running-qlk27","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:55:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-qlk27-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:55:41.932Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"f8b5bdc3-f49e-44d3-beee-708828114abf","rayCluster":"long-running-qlk27","secret":"ca-secret"}
{"level":"info","ts":"2026-07-13T15:55:41.932Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"cc32cca4-71c4-4f36-999d-14655118291a","name":"long-running-qlk27-head"}
{"level":"info","ts":"2026-07-13T15:55:41.932Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"f8b5bdc3-f49e-44d3-beee-708828114abf","rayCluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:41.933Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"199afe0e-9651-4713-aade-1ffff60d67a1","name":"test-ns-88gnn-long-running-qlk27","operation":"created"}
{"level":"info","ts":"2026-07-13T15:55:41.940Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"bc4d6513-f932-441c-bf0c-df8cf6307f6e"}
{"level":"error","ts":"2026-07-13T15:55:42.021Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"bc4d6513-f932-441c-bf0c-df8cf6307f6e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:41.940Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ce4434c9-6cf5-4e14-86a0-298108162919","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:42.021Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ce4434c9-6cf5-4e14-86a0-298108162919","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:42.021Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ce4434c9-6cf5-4e14-86a0-298108162919","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:41.941Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"f8b5bdc3-f49e-44d3-beee-708828114abf","rayCluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:41.940Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"199afe0e-9651-4713-aade-1ffff60d67a1","name":"kube-rbac-proxy-config-long-running-qlk27","operation":"created"}
{"level":"info","ts":"2026-07-13T15:55:42.021Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ce4434c9-6cf5-4e14-86a0-298108162919"}
{"level":"info","ts":"2026-07-13T15:55:42.021Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ce4434c9-6cf5-4e14-86a0-298108162919","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:55:42.021Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"4fb0dad3-aae7-4262-a0e3-7c4dd220b027"}
{"level":"info","ts":"2026-07-13T15:55:42.021Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"4fb0dad3-aae7-4262-a0e3-7c4dd220b027"}
{"level":"info","ts":"2026-07-13T15:55:42.021Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"cc32cca4-71c4-4f36-999d-14655118291a","name":"long-running-qlk27-workers"}
{"level":"info","ts":"2026-07-13T15:55:42.021Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"4fb0dad3-aae7-4262-a0e3-7c4dd220b027","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:42.021Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"cc32cca4-71c4-4f36-999d-14655118291a","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:42.021Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"4fb0dad3-aae7-4262-a0e3-7c4dd220b027","error":"head certificate secret ray-head-secret-long-running-qlk27 not found"}
{"level":"info","ts":"2026-07-13T15:55:42.022Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"448ae944-0584-4a23-ac2d-eeb76750aaa8","cluster":"long-running-qlk27"}
{"level":"error","ts":"2026-07-13T15:55:42.022Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"4fb0dad3-aae7-4262-a0e3-7c4dd220b027","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:42.022Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"4fb0dad3-aae7-4262-a0e3-7c4dd220b027","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found"}
{"level":"info","ts":"2026-07-13T15:55:42.022Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"4fb0dad3-aae7-4262-a0e3-7c4dd220b027","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:42.022Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"4fb0dad3-aae7-4262-a0e3-7c4dd220b027","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:42.022Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"4fb0dad3-aae7-4262-a0e3-7c4dd220b027","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:42.022Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"4fb0dad3-aae7-4262-a0e3-7c4dd220b027"}
{"level":"error","ts":"2026-07-13T15:55:42.022Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"4fb0dad3-aae7-4262-a0e3-7c4dd220b027","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:55:42.030Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"199afe0e-9651-4713-aade-1ffff60d67a1","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qlk27\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:42.030Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"199afe0e-9651-4713-aade-1ffff60d67a1"}
{"level":"error","ts":"2026-07-13T15:55:42.030Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"199afe0e-9651-4713-aade-1ffff60d67a1","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qlk27\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:42.030Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"226e8da2-2277-42a8-86e0-0bf85b27eee5","namespacedName":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:42.030Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"226e8da2-2277-42a8-86e0-0bf85b27eee5","mode":"IntegratedOAuth","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:42.030Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"226e8da2-2277-42a8-86e0-0bf85b27eee5","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:42.121Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"448ae944-0584-4a23-ac2d-eeb76750aaa8","name":"long-running-qlk27-head"}
{"level":"info","ts":"2026-07-13T15:55:42.121Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"cc49b31d-f6bb-4ead-9ef1-f4f39e309ece"}
{"level":"info","ts":"2026-07-13T15:55:42.121Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"cc49b31d-f6bb-4ead-9ef1-f4f39e309ece"}
{"level":"info","ts":"2026-07-13T15:55:42.122Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"cc49b31d-f6bb-4ead-9ef1-f4f39e309ece","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:42.122Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"cc49b31d-f6bb-4ead-9ef1-f4f39e309ece","error":"head certificate secret ray-head-secret-long-running-qlk27 not found"}
{"level":"error","ts":"2026-07-13T15:55:42.122Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"cc49b31d-f6bb-4ead-9ef1-f4f39e309ece","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:42.122Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"cc49b31d-f6bb-4ead-9ef1-f4f39e309ece","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found"}
{"level":"info","ts":"2026-07-13T15:55:42.122Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"cc49b31d-f6bb-4ead-9ef1-f4f39e309ece","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:42.122Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"cc49b31d-f6bb-4ead-9ef1-f4f39e309ece","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:42.122Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"cc49b31d-f6bb-4ead-9ef1-f4f39e309ece","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:42.122Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"cc49b31d-f6bb-4ead-9ef1-f4f39e309ece"}
{"level":"error","ts":"2026-07-13T15:55:42.122Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"cc49b31d-f6bb-4ead-9ef1-f4f39e309ece","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:42.123Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"226e8da2-2277-42a8-86e0-0bf85b27eee5","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:42.123Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"b8026717-70d6-4fcf-a60e-bc3027cb700e","namespacedName":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:42.123Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"b8026717-70d6-4fcf-a60e-bc3027cb700e","mode":"IntegratedOAuth","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:42.123Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"b8026717-70d6-4fcf-a60e-bc3027cb700e","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:42.124Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"f8b5bdc3-f49e-44d3-beee-708828114abf","rayCluster":"long-running-qlk27","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:55:42.132Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"f8b5bdc3-f49e-44d3-beee-708828114abf","rayCluster":"long-running-qlk27","podIPs":[]}
{"level":"error","ts":"2026-07-13T15:55:42.132Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"f8b5bdc3-f49e-44d3-beee-708828114abf","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-qlk27\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:42.132Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"f8b5bdc3-f49e-44d3-beee-708828114abf"}
{"level":"error","ts":"2026-07-13T15:55:42.132Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"f8b5bdc3-f49e-44d3-beee-708828114abf","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-qlk27\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:42.132Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"997bfb7d-95a8-4e51-8c8f-32f096de200b","rayCluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:42.221Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"448ae944-0584-4a23-ac2d-eeb76750aaa8","name":"long-running-qlk27-workers"}
{"level":"info","ts":"2026-07-13T15:55:42.321Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"448ae944-0584-4a23-ac2d-eeb76750aaa8","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:42.321Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"1caa4d64-0e26-43d5-a108-e852fc9888ec","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:42.222Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0702c9f2-4248-4695-9c02-ff832fc849f9"}
{"level":"info","ts":"2026-07-13T15:55:42.321Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"1caa4d64-0e26-43d5-a108-e852fc9888ec","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:42.321Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0702c9f2-4248-4695-9c02-ff832fc849f9"}
{"level":"info","ts":"2026-07-13T15:55:42.321Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"1caa4d64-0e26-43d5-a108-e852fc9888ec","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:42.321Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"1caa4d64-0e26-43d5-a108-e852fc9888ec"}
{"level":"info","ts":"2026-07-13T15:55:42.321Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"1caa4d64-0e26-43d5-a108-e852fc9888ec","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:55:42.321Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0702c9f2-4248-4695-9c02-ff832fc849f9","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:42.321Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0702c9f2-4248-4695-9c02-ff832fc849f9","error":"head certificate secret ray-head-secret-long-running-qlk27 not found"}
{"level":"error","ts":"2026-07-13T15:55:42.321Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0702c9f2-4248-4695-9c02-ff832fc849f9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:42.322Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0702c9f2-4248-4695-9c02-ff832fc849f9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found"}
{"level":"info","ts":"2026-07-13T15:55:42.322Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0702c9f2-4248-4695-9c02-ff832fc849f9","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:42.321Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"2d43d64f-0ccb-46f2-9ee9-7d3d9a332551","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:42.223Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"b8026717-70d6-4fcf-a60e-bc3027cb700e","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:42.322Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0702c9f2-4248-4695-9c02-ff832fc849f9","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"error","ts":"2026-07-13T15:55:42.322Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"997bfb7d-95a8-4e51-8c8f-32f096de200b","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-qlk27\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:55:42.421Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"997bfb7d-95a8-4e51-8c8f-32f096de200b","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-qlk27\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:42.421Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0702c9f2-4248-4695-9c02-ff832fc849f9","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:42.421Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"997bfb7d-95a8-4e51-8c8f-32f096de200b"}
{"level":"info","ts":"2026-07-13T15:55:42.421Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0702c9f2-4248-4695-9c02-ff832fc849f9"}
{"level":"error","ts":"2026-07-13T15:55:42.421Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"997bfb7d-95a8-4e51-8c8f-32f096de200b","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-qlk27\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:55:42.422Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0702c9f2-4248-4695-9c02-ff832fc849f9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:42.422Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"e037eec3-8c4d-4051-9f30-83db7c66f198","rayCluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:42.422Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"e037eec3-8c4d-4051-9f30-83db7c66f198"}
{"level":"info","ts":"2026-07-13T15:55:42.431Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"2d43d64f-0ccb-46f2-9ee9-7d3d9a332551","name":"long-running-qlk27-head"}
{"level":"info","ts":"2026-07-13T15:55:42.432Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"1fad2e57-785b-4750-b235-df3d92a0dca2","rayCluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:42.432Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"1fad2e57-785b-4750-b235-df3d92a0dca2"}
{"level":"info","ts":"2026-07-13T15:55:42.441Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"2d43d64f-0ccb-46f2-9ee9-7d3d9a332551","name":"long-running-qlk27-workers"}
{"level":"info","ts":"2026-07-13T15:55:42.441Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"2d43d64f-0ccb-46f2-9ee9-7d3d9a332551","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:42.502Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6396d3f1-43a8-428d-a264-12ed53cbe0ec"}
{"level":"info","ts":"2026-07-13T15:55:42.502Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6396d3f1-43a8-428d-a264-12ed53cbe0ec"}
{"level":"info","ts":"2026-07-13T15:55:42.502Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6396d3f1-43a8-428d-a264-12ed53cbe0ec","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:42.502Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6396d3f1-43a8-428d-a264-12ed53cbe0ec","error":"head certificate secret ray-head-secret-long-running-qlk27 not found"}
{"level":"error","ts":"2026-07-13T15:55:42.502Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6396d3f1-43a8-428d-a264-12ed53cbe0ec","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:42.502Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6396d3f1-43a8-428d-a264-12ed53cbe0ec","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found"}
{"level":"info","ts":"2026-07-13T15:55:42.502Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6396d3f1-43a8-428d-a264-12ed53cbe0ec","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:42.502Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6396d3f1-43a8-428d-a264-12ed53cbe0ec","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:42.502Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6396d3f1-43a8-428d-a264-12ed53cbe0ec","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:42.502Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6396d3f1-43a8-428d-a264-12ed53cbe0ec"}
{"level":"error","ts":"2026-07-13T15:55:42.502Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6396d3f1-43a8-428d-a264-12ed53cbe0ec","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:42.663Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a398ac31-c2ab-460f-a60d-d1ca0cf78fc6"}
{"level":"info","ts":"2026-07-13T15:55:42.663Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a398ac31-c2ab-460f-a60d-d1ca0cf78fc6"}
{"level":"info","ts":"2026-07-13T15:55:42.663Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a398ac31-c2ab-460f-a60d-d1ca0cf78fc6","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:42.663Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a398ac31-c2ab-460f-a60d-d1ca0cf78fc6","error":"head certificate secret ray-head-secret-long-running-qlk27 not found"}
{"level":"error","ts":"2026-07-13T15:55:42.663Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a398ac31-c2ab-460f-a60d-d1ca0cf78fc6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:42.663Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a398ac31-c2ab-460f-a60d-d1ca0cf78fc6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found"}
{"level":"info","ts":"2026-07-13T15:55:42.663Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a398ac31-c2ab-460f-a60d-d1ca0cf78fc6","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:42.663Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a398ac31-c2ab-460f-a60d-d1ca0cf78fc6","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:42.663Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a398ac31-c2ab-460f-a60d-d1ca0cf78fc6","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:42.663Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a398ac31-c2ab-460f-a60d-d1ca0cf78fc6"}
{"level":"error","ts":"2026-07-13T15:55:42.663Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a398ac31-c2ab-460f-a60d-d1ca0cf78fc6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:42.983Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"5147a5f0-a117-40e4-8595-f4858f30ae98"}
{"level":"info","ts":"2026-07-13T15:55:42.983Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"5147a5f0-a117-40e4-8595-f4858f30ae98"}
{"level":"info","ts":"2026-07-13T15:55:42.983Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"5147a5f0-a117-40e4-8595-f4858f30ae98","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:42.984Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"5147a5f0-a117-40e4-8595-f4858f30ae98","error":"head certificate secret ray-head-secret-long-running-qlk27 not found"}
{"level":"error","ts":"2026-07-13T15:55:42.984Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"5147a5f0-a117-40e4-8595-f4858f30ae98","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:42.984Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"5147a5f0-a117-40e4-8595-f4858f30ae98","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found"}
{"level":"info","ts":"2026-07-13T15:55:42.984Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"5147a5f0-a117-40e4-8595-f4858f30ae98","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:42.984Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"5147a5f0-a117-40e4-8595-f4858f30ae98","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:42.984Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"5147a5f0-a117-40e4-8595-f4858f30ae98","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:42.984Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"5147a5f0-a117-40e4-8595-f4858f30ae98"}
{"level":"error","ts":"2026-07-13T15:55:42.984Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"5147a5f0-a117-40e4-8595-f4858f30ae98","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:43.625Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ba9379aa-3b58-4e73-8458-67fefac1259b"}
{"level":"info","ts":"2026-07-13T15:55:43.625Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ba9379aa-3b58-4e73-8458-67fefac1259b"}
{"level":"info","ts":"2026-07-13T15:55:43.625Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ba9379aa-3b58-4e73-8458-67fefac1259b","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:43.625Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ba9379aa-3b58-4e73-8458-67fefac1259b","error":"head certificate secret ray-head-secret-long-running-qlk27 not found"}
{"level":"error","ts":"2026-07-13T15:55:43.625Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ba9379aa-3b58-4e73-8458-67fefac1259b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:43.625Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ba9379aa-3b58-4e73-8458-67fefac1259b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found"}
{"level":"info","ts":"2026-07-13T15:55:43.625Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ba9379aa-3b58-4e73-8458-67fefac1259b","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:43.625Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ba9379aa-3b58-4e73-8458-67fefac1259b","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:43.625Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ba9379aa-3b58-4e73-8458-67fefac1259b","filter labels":["test-ns-88gnn",{"ray.io/cluster":"long-running-qlk27","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:55:43.625Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ba9379aa-3b58-4e73-8458-67fefac1259b"}
{"level":"error","ts":"2026-07-13T15:55:43.625Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ba9379aa-3b58-4e73-8458-67fefac1259b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qlk27 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:55:44.058Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a6855ce8-6c4a-433a-88af-713d1da29799","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:44.058Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a6855ce8-6c4a-433a-88af-713d1da29799","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:44.058Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a6855ce8-6c4a-433a-88af-713d1da29799","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:44.058Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a6855ce8-6c4a-433a-88af-713d1da29799"}
{"level":"info","ts":"2026-07-13T15:55:44.058Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a6855ce8-6c4a-433a-88af-713d1da29799","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:55:44.906Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67"}
{"level":"info","ts":"2026-07-13T15:55:44.906Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67"}
{"level":"info","ts":"2026-07-13T15:55:44.906Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:44.906Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67"}
{"level":"info","ts":"2026-07-13T15:55:44.906Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:55:44.906Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67"}
{"level":"info","ts":"2026-07-13T15:55:44.906Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","mode":"IntegratedOAuth","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:44.906Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","cluster":"long-running-qlk27","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:55:44.906Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","cluster":"long-running-qlk27","authType":"IntegratedOAuth","serviceAccount":"long-running-qlk27-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T15:55:44.906Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-qlk27","ray.io/group":"headgroup","ray.io/identifier":"long-running-qlk27-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T15:55:44.906Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-qlk27-head-svc.test-ns-88gnn.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T15:55:44.906Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-qlk27-head-svc.test-ns-88gnn.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:55:44.906Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-qlk27-head-svc.test-ns-88gnn.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:55:44.906Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","enabled":true}
{"level":"info","ts":"2026-07-13T15:55:44.931Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","name":"long-running-qlk27-head-xnt9s"}
{"level":"info","ts":"2026-07-13T15:55:44.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:55:44.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:44.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T15:55:44.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:44.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T15:55:44.931Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67"}
{"level":"info","ts":"2026-07-13T15:55:44.931Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","nodeType":"worker","rayStartParams":{"address":"long-running-qlk27-head-svc.test-ns-88gnn.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T15:55:44.931Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","rayStartCmd":"ray start  --address=long-running-qlk27-head-svc.test-ns-88gnn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:55:44.931Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-qlk27-head-svc.test-ns-88gnn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:55:44.931Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","enabled":true}
{"level":"info","ts":"2026-07-13T15:55:44.949Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","name":"long-running-qlk27-small-group-worker-tf68l"}
{"level":"info","ts":"2026-07-13T15:55:44.949Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","podName":"long-running-qlk27-head-xnt9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:44.950Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-qlk27-head-svc"},"newHead":{"podName":"long-running-qlk27-head-xnt9s","serviceName":"long-running-qlk27-head-svc"}}
{"level":"info","ts":"2026-07-13T15:55:44.950Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aad04e4b-e31c-4c18-8e27-1d081c4f0c67","name":"long-running-qlk27","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:55:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-qlk27-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:55:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:55:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qlk27-head-xnt9s","serviceName":"long-running-qlk27-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:55:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:55:44.965Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"dd169521-a28e-48d1-b1b4-8046acb87789","rayCluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:44.965Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"d69e7e2d-4193-4ef2-83a4-e36fc88a5db9","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:44.965Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0919fa18-3158-41e6-b388-8ac8fb2bc4db"}
{"level":"info","ts":"2026-07-13T15:55:44.965Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0919fa18-3158-41e6-b388-8ac8fb2bc4db"}
{"level":"info","ts":"2026-07-13T15:55:44.965Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0919fa18-3158-41e6-b388-8ac8fb2bc4db","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:44.965Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"dd169521-a28e-48d1-b1b4-8046acb87789","rayCluster":"long-running-qlk27","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:55:44.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0919fa18-3158-41e6-b388-8ac8fb2bc4db","Found 1 head Pod":"long-running-qlk27-head-xnt9s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:55:44.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0919fa18-3158-41e6-b388-8ac8fb2bc4db","head Pod":"long-running-qlk27-head-xnt9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qlk27-head-xnt9s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:44.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0919fa18-3158-41e6-b388-8ac8fb2bc4db","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:55:44.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0919fa18-3158-41e6-b388-8ac8fb2bc4db","worker Pod":"long-running-qlk27-small-group-worker-tf68l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qlk27-small-group-worker-tf68l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:44.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0919fa18-3158-41e6-b388-8ac8fb2bc4db","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:44.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0919fa18-3158-41e6-b388-8ac8fb2bc4db","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:55:44.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0919fa18-3158-41e6-b388-8ac8fb2bc4db","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:44.966Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0919fa18-3158-41e6-b388-8ac8fb2bc4db","podName":"long-running-qlk27-head-xnt9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:44.966Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0919fa18-3158-41e6-b388-8ac8fb2bc4db","podName":"long-running-qlk27-head-xnt9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:44.966Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0919fa18-3158-41e6-b388-8ac8fb2bc4db","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:55:44.966Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0919fa18-3158-41e6-b388-8ac8fb2bc4db","seconds":300}
{"level":"info","ts":"2026-07-13T15:55:44.965Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d0ca9303-6eb1-45a9-a39e-2c9f6fae82da","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:44.966Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d0ca9303-6eb1-45a9-a39e-2c9f6fae82da","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:44.966Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d0ca9303-6eb1-45a9-a39e-2c9f6fae82da","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:44.966Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d0ca9303-6eb1-45a9-a39e-2c9f6fae82da"}
{"level":"info","ts":"2026-07-13T15:55:44.966Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d0ca9303-6eb1-45a9-a39e-2c9f6fae82da","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:55:44.974Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"09707c26-36ae-4b83-bed3-8abe50947950"}
{"level":"info","ts":"2026-07-13T15:55:44.974Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"09707c26-36ae-4b83-bed3-8abe50947950"}
{"level":"info","ts":"2026-07-13T15:55:44.974Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"09707c26-36ae-4b83-bed3-8abe50947950","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:44.974Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"09707c26-36ae-4b83-bed3-8abe50947950","Found 1 head Pod":"long-running-qlk27-head-xnt9s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:55:44.974Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"09707c26-36ae-4b83-bed3-8abe50947950","head Pod":"long-running-qlk27-head-xnt9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qlk27-head-xnt9s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:44.974Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"09707c26-36ae-4b83-bed3-8abe50947950","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:55:44.974Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"09707c26-36ae-4b83-bed3-8abe50947950","worker Pod":"long-running-qlk27-small-group-worker-tf68l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qlk27-small-group-worker-tf68l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:44.974Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"09707c26-36ae-4b83-bed3-8abe50947950","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:44.974Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"09707c26-36ae-4b83-bed3-8abe50947950","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:55:44.974Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"09707c26-36ae-4b83-bed3-8abe50947950","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:44.974Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"09707c26-36ae-4b83-bed3-8abe50947950","podName":"long-running-qlk27-head-xnt9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:44.974Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"09707c26-36ae-4b83-bed3-8abe50947950","podName":"long-running-qlk27-head-xnt9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:44.974Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"09707c26-36ae-4b83-bed3-8abe50947950","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:55:44.974Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"09707c26-36ae-4b83-bed3-8abe50947950","seconds":300}
{"level":"info","ts":"2026-07-13T15:55:44.978Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0c8a25c4-50d7-46ef-8677-e655dc959532"}
{"level":"info","ts":"2026-07-13T15:55:44.978Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0c8a25c4-50d7-46ef-8677-e655dc959532"}
{"level":"info","ts":"2026-07-13T15:55:44.978Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0c8a25c4-50d7-46ef-8677-e655dc959532","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:44.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0c8a25c4-50d7-46ef-8677-e655dc959532","Found 1 head Pod":"long-running-qlk27-head-xnt9s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:55:44.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0c8a25c4-50d7-46ef-8677-e655dc959532","head Pod":"long-running-qlk27-head-xnt9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qlk27-head-xnt9s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:44.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0c8a25c4-50d7-46ef-8677-e655dc959532","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:55:44.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0c8a25c4-50d7-46ef-8677-e655dc959532","worker Pod":"long-running-qlk27-small-group-worker-tf68l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qlk27-small-group-worker-tf68l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:44.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0c8a25c4-50d7-46ef-8677-e655dc959532","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:44.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0c8a25c4-50d7-46ef-8677-e655dc959532","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:55:44.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0c8a25c4-50d7-46ef-8677-e655dc959532","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:44.978Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0c8a25c4-50d7-46ef-8677-e655dc959532","podName":"long-running-qlk27-head-xnt9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:44.978Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0c8a25c4-50d7-46ef-8677-e655dc959532","podName":"long-running-qlk27-head-xnt9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:44.978Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0c8a25c4-50d7-46ef-8677-e655dc959532","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:55:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:55:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:55:44.978Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"0c8a25c4-50d7-46ef-8677-e655dc959532","name":"long-running-qlk27","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:55:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qlk27-head-xnt9s","serviceName":"long-running-qlk27-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:55:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:55:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qlk27-head-xnt9s","serviceName":"long-running-qlk27-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:55:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:55:44.978Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"d69e7e2d-4193-4ef2-83a4-e36fc88a5db9","name":"long-running-qlk27-head"}
{"level":"info","ts":"2026-07-13T15:55:45.022Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"cba9c192-944e-448f-a1c4-72f7885a46e7","rayCluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:45.022Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"c9ef0568-0f42-42bd-82fe-3629f32e2867"}
{"level":"info","ts":"2026-07-13T15:55:45.022Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"c9ef0568-0f42-42bd-82fe-3629f32e2867"}
{"level":"info","ts":"2026-07-13T15:55:45.022Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"c9ef0568-0f42-42bd-82fe-3629f32e2867","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:45.022Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2d4c0531-f525-4877-b124-70a65352a01d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:45.022Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2d4c0531-f525-4877-b124-70a65352a01d","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:45.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"c9ef0568-0f42-42bd-82fe-3629f32e2867","Found 1 head Pod":"long-running-qlk27-head-xnt9s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:55:45.022Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2d4c0531-f525-4877-b124-70a65352a01d","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:45.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"c9ef0568-0f42-42bd-82fe-3629f32e2867","head Pod":"long-running-qlk27-head-xnt9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qlk27-head-xnt9s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:45.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"c9ef0568-0f42-42bd-82fe-3629f32e2867","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:55:45.022Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"cba9c192-944e-448f-a1c4-72f7885a46e7","rayCluster":"long-running-qlk27","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:55:45.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"c9ef0568-0f42-42bd-82fe-3629f32e2867","worker Pod":"long-running-qlk27-small-group-worker-tf68l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qlk27-small-group-worker-tf68l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:45.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"c9ef0568-0f42-42bd-82fe-3629f32e2867","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:45.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"c9ef0568-0f42-42bd-82fe-3629f32e2867","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:55:45.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"c9ef0568-0f42-42bd-82fe-3629f32e2867","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:45.022Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2d4c0531-f525-4877-b124-70a65352a01d"}
{"level":"info","ts":"2026-07-13T15:55:45.022Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2d4c0531-f525-4877-b124-70a65352a01d","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:55:45.022Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"c9ef0568-0f42-42bd-82fe-3629f32e2867","podName":"long-running-qlk27-head-xnt9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:45.022Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"c9ef0568-0f42-42bd-82fe-3629f32e2867","podName":"long-running-qlk27-head-xnt9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:45.022Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"c9ef0568-0f42-42bd-82fe-3629f32e2867","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:55:45.022Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"c9ef0568-0f42-42bd-82fe-3629f32e2867","seconds":300}
{"level":"info","ts":"2026-07-13T15:55:45.025Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"d69e7e2d-4193-4ef2-83a4-e36fc88a5db9","name":"long-running-qlk27-workers"}
{"level":"info","ts":"2026-07-13T15:55:45.025Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"d69e7e2d-4193-4ef2-83a4-e36fc88a5db9","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:45.026Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"117aca6d-23e1-4f28-be9c-616baccff94b","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:45.038Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"117aca6d-23e1-4f28-be9c-616baccff94b","name":"long-running-qlk27-head"}
{"level":"info","ts":"2026-07-13T15:55:45.046Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"117aca6d-23e1-4f28-be9c-616baccff94b","name":"long-running-qlk27-workers"}
{"level":"info","ts":"2026-07-13T15:55:45.046Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"117aca6d-23e1-4f28-be9c-616baccff94b","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:45.394Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6cd0c141-7059-4a0f-a3ad-f4bc8cec465b"}
{"level":"info","ts":"2026-07-13T15:55:45.394Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6cd0c141-7059-4a0f-a3ad-f4bc8cec465b"}
{"level":"info","ts":"2026-07-13T15:55:45.394Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6cd0c141-7059-4a0f-a3ad-f4bc8cec465b","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:45.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6cd0c141-7059-4a0f-a3ad-f4bc8cec465b","Found 1 head Pod":"long-running-qlk27-head-xnt9s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:55:45.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6cd0c141-7059-4a0f-a3ad-f4bc8cec465b","head Pod":"long-running-qlk27-head-xnt9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qlk27-head-xnt9s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:45.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6cd0c141-7059-4a0f-a3ad-f4bc8cec465b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:55:45.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6cd0c141-7059-4a0f-a3ad-f4bc8cec465b","worker Pod":"long-running-qlk27-small-group-worker-tf68l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qlk27-small-group-worker-tf68l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:45.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6cd0c141-7059-4a0f-a3ad-f4bc8cec465b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:45.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6cd0c141-7059-4a0f-a3ad-f4bc8cec465b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:55:45.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6cd0c141-7059-4a0f-a3ad-f4bc8cec465b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:45.394Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6cd0c141-7059-4a0f-a3ad-f4bc8cec465b","podName":"long-running-qlk27-head-xnt9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:45.394Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6cd0c141-7059-4a0f-a3ad-f4bc8cec465b","podName":"long-running-qlk27-head-xnt9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:45.394Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6cd0c141-7059-4a0f-a3ad-f4bc8cec465b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:55:45.394Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"6cd0c141-7059-4a0f-a3ad-f4bc8cec465b","seconds":300}
{"level":"info","ts":"2026-07-13T15:55:45.412Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"499b3de5-7ef6-42f0-94ec-ee2aefcc3642"}
{"level":"info","ts":"2026-07-13T15:55:45.412Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"499b3de5-7ef6-42f0-94ec-ee2aefcc3642"}
{"level":"info","ts":"2026-07-13T15:55:45.412Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"499b3de5-7ef6-42f0-94ec-ee2aefcc3642","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:45.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"499b3de5-7ef6-42f0-94ec-ee2aefcc3642","Found 1 head Pod":"long-running-qlk27-head-xnt9s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:55:45.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"499b3de5-7ef6-42f0-94ec-ee2aefcc3642","head Pod":"long-running-qlk27-head-xnt9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qlk27-head-xnt9s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:45.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"499b3de5-7ef6-42f0-94ec-ee2aefcc3642","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:55:45.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"499b3de5-7ef6-42f0-94ec-ee2aefcc3642","worker Pod":"long-running-qlk27-small-group-worker-tf68l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qlk27-small-group-worker-tf68l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:45.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"499b3de5-7ef6-42f0-94ec-ee2aefcc3642","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:45.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"499b3de5-7ef6-42f0-94ec-ee2aefcc3642","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:55:45.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"499b3de5-7ef6-42f0-94ec-ee2aefcc3642","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:45.412Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"499b3de5-7ef6-42f0-94ec-ee2aefcc3642","podName":"long-running-qlk27-head-xnt9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:45.412Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"499b3de5-7ef6-42f0-94ec-ee2aefcc3642","podName":"long-running-qlk27-head-xnt9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:45.412Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"499b3de5-7ef6-42f0-94ec-ee2aefcc3642","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:55:45.412Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"499b3de5-7ef6-42f0-94ec-ee2aefcc3642","seconds":300}
{"level":"info","ts":"2026-07-13T15:55:46.217Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ff6aaecb-1c60-4bb6-9a0e-e02a1ba432ac"}
{"level":"info","ts":"2026-07-13T15:55:46.218Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ff6aaecb-1c60-4bb6-9a0e-e02a1ba432ac"}
{"level":"info","ts":"2026-07-13T15:55:46.218Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ff6aaecb-1c60-4bb6-9a0e-e02a1ba432ac","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:46.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ff6aaecb-1c60-4bb6-9a0e-e02a1ba432ac","Found 1 head Pod":"long-running-qlk27-head-xnt9s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:55:46.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ff6aaecb-1c60-4bb6-9a0e-e02a1ba432ac","head Pod":"long-running-qlk27-head-xnt9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qlk27-head-xnt9s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:46.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ff6aaecb-1c60-4bb6-9a0e-e02a1ba432ac","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:55:46.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ff6aaecb-1c60-4bb6-9a0e-e02a1ba432ac","worker Pod":"long-running-qlk27-small-group-worker-tf68l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qlk27-small-group-worker-tf68l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:46.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ff6aaecb-1c60-4bb6-9a0e-e02a1ba432ac","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:46.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ff6aaecb-1c60-4bb6-9a0e-e02a1ba432ac","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:55:46.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ff6aaecb-1c60-4bb6-9a0e-e02a1ba432ac","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:46.218Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ff6aaecb-1c60-4bb6-9a0e-e02a1ba432ac","podName":"long-running-qlk27-small-group-worker-tf68l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:46.218Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ff6aaecb-1c60-4bb6-9a0e-e02a1ba432ac","podName":"long-running-qlk27-small-group-worker-tf68l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:46.218Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ff6aaecb-1c60-4bb6-9a0e-e02a1ba432ac","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"long-running-qlk27-head-xnt9s","serviceName":"long-running-qlk27-head-svc"},"newHead":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"long-running-qlk27-head-xnt9s","serviceName":"long-running-qlk27-head-svc"}}
{"level":"info","ts":"2026-07-13T15:55:46.218Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ff6aaecb-1c60-4bb6-9a0e-e02a1ba432ac","name":"long-running-qlk27","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:55:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qlk27-head-xnt9s","serviceName":"long-running-qlk27-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:55:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:55:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"long-running-qlk27-head-xnt9s","serviceName":"long-running-qlk27-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:55:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:55:46.242Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"c630758b-fec4-471d-aaa7-cfd70b0eeab6","rayCluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:46.242Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"c2815b0d-4b09-4a54-be9d-2bab3dd8c211","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:46.242Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f0f9bfd6-89d7-4a79-8be6-93fa184906c8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:46.242Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f0f9bfd6-89d7-4a79-8be6-93fa184906c8","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:46.242Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f0f9bfd6-89d7-4a79-8be6-93fa184906c8","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:46.242Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"c630758b-fec4-471d-aaa7-cfd70b0eeab6","pod":"long-running-qlk27-head-xnt9s","ip":"10.133.0.24"}
{"level":"info","ts":"2026-07-13T15:55:46.243Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f0f9bfd6-89d7-4a79-8be6-93fa184906c8"}
{"level":"info","ts":"2026-07-13T15:55:46.243Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f0f9bfd6-89d7-4a79-8be6-93fa184906c8","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:55:46.266Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"c2815b0d-4b09-4a54-be9d-2bab3dd8c211","name":"long-running-qlk27-head"}
{"level":"info","ts":"2026-07-13T15:55:46.267Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"c630758b-fec4-471d-aaa7-cfd70b0eeab6","certificate":"ray-head-cert-long-running-qlk27","dnsNames":["long-running-qlk27-head-svc","localhost","long-running-qlk27-head-svc.test-ns-88gnn.svc","long-running-qlk27-head-svc.test-ns-88gnn.svc.cluster.local"],"ipAddresses":["10.133.0.24","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:55:46.289Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"c2815b0d-4b09-4a54-be9d-2bab3dd8c211","name":"long-running-qlk27-workers"}
{"level":"info","ts":"2026-07-13T15:55:46.289Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"c2815b0d-4b09-4a54-be9d-2bab3dd8c211","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:46.290Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"c630758b-fec4-471d-aaa7-cfd70b0eeab6","certificate":"ray-worker-cert-long-running-qlk27","dnsNames":["long-running-qlk27-worker-svc","localhost","long-running-qlk27-small-group","long-running-qlk27-worker-svc.test-ns-88gnn.svc","long-running-qlk27-worker-svc.test-ns-88gnn.svc.cluster.local","*.long-running-qlk27-worker-svc.test-ns-88gnn.svc","*.long-running-qlk27-worker-svc.test-ns-88gnn.svc.cluster.local","*-worker-*.test-ns-88gnn.svc","*-worker-*.test-ns-88gnn.svc.cluster.local"],"ipAddresses":["10.133.0.24","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:55:46.290Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"c630758b-fec4-471d-aaa7-cfd70b0eeab6","rayCluster":"long-running-qlk27","podIPs":["10.133.0.24"]}
{"level":"info","ts":"2026-07-13T15:55:46.408Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"54e575fb-64ad-4606-bb28-e03cb88820ae"}
{"level":"info","ts":"2026-07-13T15:55:46.408Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"54e575fb-64ad-4606-bb28-e03cb88820ae"}
{"level":"info","ts":"2026-07-13T15:55:46.408Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"54e575fb-64ad-4606-bb28-e03cb88820ae","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:46.408Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"54e575fb-64ad-4606-bb28-e03cb88820ae","Found 1 head Pod":"long-running-qlk27-head-xnt9s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:55:46.408Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"54e575fb-64ad-4606-bb28-e03cb88820ae","head Pod":"long-running-qlk27-head-xnt9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qlk27-head-xnt9s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:46.408Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"54e575fb-64ad-4606-bb28-e03cb88820ae","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:55:46.408Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"54e575fb-64ad-4606-bb28-e03cb88820ae","worker Pod":"long-running-qlk27-small-group-worker-tf68l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qlk27-small-group-worker-tf68l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:46.408Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"54e575fb-64ad-4606-bb28-e03cb88820ae","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:46.408Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"54e575fb-64ad-4606-bb28-e03cb88820ae","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:55:46.408Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"54e575fb-64ad-4606-bb28-e03cb88820ae","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:46.408Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"54e575fb-64ad-4606-bb28-e03cb88820ae","podName":"long-running-qlk27-head-xnt9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:46.408Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"54e575fb-64ad-4606-bb28-e03cb88820ae","podName":"long-running-qlk27-head-xnt9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:46.408Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"54e575fb-64ad-4606-bb28-e03cb88820ae","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:55:46.408Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"54e575fb-64ad-4606-bb28-e03cb88820ae","seconds":300}
{"level":"info","ts":"2026-07-13T15:55:46.966Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"8a5f0cef-260f-4ece-9943-54e84a8cec7b"}
{"level":"info","ts":"2026-07-13T15:55:46.966Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"8a5f0cef-260f-4ece-9943-54e84a8cec7b"}
{"level":"info","ts":"2026-07-13T15:55:46.966Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"8a5f0cef-260f-4ece-9943-54e84a8cec7b","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:46.966Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"8a5f0cef-260f-4ece-9943-54e84a8cec7b","Found 1 head Pod":"long-running-qlk27-head-xnt9s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:55:46.966Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"8a5f0cef-260f-4ece-9943-54e84a8cec7b","head Pod":"long-running-qlk27-head-xnt9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qlk27-head-xnt9s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:46.966Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"8a5f0cef-260f-4ece-9943-54e84a8cec7b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:55:46.966Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"8a5f0cef-260f-4ece-9943-54e84a8cec7b","worker Pod":"long-running-qlk27-small-group-worker-tf68l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qlk27-small-group-worker-tf68l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:46.966Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"8a5f0cef-260f-4ece-9943-54e84a8cec7b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:46.966Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"8a5f0cef-260f-4ece-9943-54e84a8cec7b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:55:46.966Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"8a5f0cef-260f-4ece-9943-54e84a8cec7b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:46.966Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"8a5f0cef-260f-4ece-9943-54e84a8cec7b","podName":"long-running-qlk27-head-xnt9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:46.966Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"8a5f0cef-260f-4ece-9943-54e84a8cec7b","podName":"long-running-qlk27-head-xnt9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:46.966Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"8a5f0cef-260f-4ece-9943-54e84a8cec7b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:55:46.966Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"8a5f0cef-260f-4ece-9943-54e84a8cec7b","seconds":300}
{"level":"info","ts":"2026-07-13T15:55:47.059Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"3eb80986-0ab4-40e1-a7f8-57a55ed462de","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:47.059Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"3eb80986-0ab4-40e1-a7f8-57a55ed462de","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:47.059Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"3eb80986-0ab4-40e1-a7f8-57a55ed462de","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:47.059Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"3eb80986-0ab4-40e1-a7f8-57a55ed462de"}
{"level":"info","ts":"2026-07-13T15:55:47.059Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"3eb80986-0ab4-40e1-a7f8-57a55ed462de","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:55:50.060Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d79e1dfb-fd06-4a97-8cbb-1e1420f81889","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:50.060Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d79e1dfb-fd06-4a97-8cbb-1e1420f81889","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:50.060Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d79e1dfb-fd06-4a97-8cbb-1e1420f81889","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:50.060Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d79e1dfb-fd06-4a97-8cbb-1e1420f81889"}
{"level":"info","ts":"2026-07-13T15:55:50.060Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d79e1dfb-fd06-4a97-8cbb-1e1420f81889","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:55:51.243Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2ab811f7-72c3-4e21-a69f-a2dc244b1dc9"}
{"level":"info","ts":"2026-07-13T15:55:51.243Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2ab811f7-72c3-4e21-a69f-a2dc244b1dc9"}
{"level":"info","ts":"2026-07-13T15:55:51.243Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2ab811f7-72c3-4e21-a69f-a2dc244b1dc9","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:51.243Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2ab811f7-72c3-4e21-a69f-a2dc244b1dc9","Found 1 head Pod":"long-running-qlk27-head-xnt9s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:55:51.243Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2ab811f7-72c3-4e21-a69f-a2dc244b1dc9","head Pod":"long-running-qlk27-head-xnt9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qlk27-head-xnt9s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:51.243Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2ab811f7-72c3-4e21-a69f-a2dc244b1dc9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:55:51.243Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2ab811f7-72c3-4e21-a69f-a2dc244b1dc9","worker Pod":"long-running-qlk27-small-group-worker-tf68l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qlk27-small-group-worker-tf68l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:51.243Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2ab811f7-72c3-4e21-a69f-a2dc244b1dc9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:51.243Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2ab811f7-72c3-4e21-a69f-a2dc244b1dc9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:55:51.243Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2ab811f7-72c3-4e21-a69f-a2dc244b1dc9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:51.243Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2ab811f7-72c3-4e21-a69f-a2dc244b1dc9","podName":"long-running-qlk27-head-xnt9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:51.243Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2ab811f7-72c3-4e21-a69f-a2dc244b1dc9","podName":"long-running-qlk27-head-xnt9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:51.243Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2ab811f7-72c3-4e21-a69f-a2dc244b1dc9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:55:51.243Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2ab811f7-72c3-4e21-a69f-a2dc244b1dc9","seconds":300}
{"level":"info","ts":"2026-07-13T15:55:52.250Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"841becae-2ccb-4b43-a65e-31ba16c3b9e1"}
{"level":"info","ts":"2026-07-13T15:55:52.250Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"841becae-2ccb-4b43-a65e-31ba16c3b9e1"}
{"level":"info","ts":"2026-07-13T15:55:52.250Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"841becae-2ccb-4b43-a65e-31ba16c3b9e1","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:52.250Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"841becae-2ccb-4b43-a65e-31ba16c3b9e1","Found 1 head Pod":"long-running-qlk27-head-xnt9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:55:52.250Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"841becae-2ccb-4b43-a65e-31ba16c3b9e1","head Pod":"long-running-qlk27-head-xnt9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qlk27-head-xnt9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:52.250Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"841becae-2ccb-4b43-a65e-31ba16c3b9e1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:55:52.250Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"841becae-2ccb-4b43-a65e-31ba16c3b9e1","worker Pod":"long-running-qlk27-small-group-worker-tf68l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qlk27-small-group-worker-tf68l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:52.250Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"841becae-2ccb-4b43-a65e-31ba16c3b9e1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:52.250Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"841becae-2ccb-4b43-a65e-31ba16c3b9e1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:55:52.250Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"841becae-2ccb-4b43-a65e-31ba16c3b9e1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:52.250Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"841becae-2ccb-4b43-a65e-31ba16c3b9e1","podName":"long-running-qlk27-head-xnt9s","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:55:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:55:52.251Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"841becae-2ccb-4b43-a65e-31ba16c3b9e1","podName":"long-running-qlk27-head-xnt9s","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:55:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:55:52.251Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"841becae-2ccb-4b43-a65e-31ba16c3b9e1","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:55:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:55:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:55:52.251Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"841becae-2ccb-4b43-a65e-31ba16c3b9e1","name":"long-running-qlk27","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:55:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"long-running-qlk27-head-xnt9s","serviceName":"long-running-qlk27-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:55:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:55:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"long-running-qlk27-head-xnt9s","serviceName":"long-running-qlk27-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:55:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:55:52.263Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a1a2879b-852a-4d84-9837-32e2653264a4","rayCluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:52.263Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a90b8d35-cfcb-446a-8505-cb8cbc4e3da5","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:52.263Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"238509b9-5116-4139-96c2-ffb925abf36e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:52.263Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a1a2879b-852a-4d84-9837-32e2653264a4","pod":"long-running-qlk27-head-xnt9s","ip":"10.133.0.24"}
{"level":"info","ts":"2026-07-13T15:55:52.263Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"238509b9-5116-4139-96c2-ffb925abf36e","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:52.263Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a1a2879b-852a-4d84-9837-32e2653264a4","pod":"long-running-qlk27-small-group-worker-tf68l","ip":"10.134.0.29"}
{"level":"info","ts":"2026-07-13T15:55:52.263Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"238509b9-5116-4139-96c2-ffb925abf36e","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:52.263Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"238509b9-5116-4139-96c2-ffb925abf36e"}
{"level":"info","ts":"2026-07-13T15:55:52.263Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"238509b9-5116-4139-96c2-ffb925abf36e","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:55:52.273Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a90b8d35-cfcb-446a-8505-cb8cbc4e3da5","name":"long-running-qlk27-head"}
{"level":"info","ts":"2026-07-13T15:55:52.274Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a1a2879b-852a-4d84-9837-32e2653264a4","certificate":"ray-head-cert-long-running-qlk27","dnsNames":["long-running-qlk27-head-svc","localhost","long-running-qlk27-head-svc.test-ns-88gnn.svc","long-running-qlk27-head-svc.test-ns-88gnn.svc.cluster.local"],"ipAddresses":["10.133.0.24","10.134.0.29","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:55:52.284Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a90b8d35-cfcb-446a-8505-cb8cbc4e3da5","name":"long-running-qlk27-workers"}
{"level":"info","ts":"2026-07-13T15:55:52.284Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a90b8d35-cfcb-446a-8505-cb8cbc4e3da5","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:52.285Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a1a2879b-852a-4d84-9837-32e2653264a4","certificate":"ray-worker-cert-long-running-qlk27","dnsNames":["long-running-qlk27-worker-svc","localhost","long-running-qlk27-small-group","long-running-qlk27-worker-svc.test-ns-88gnn.svc","long-running-qlk27-worker-svc.test-ns-88gnn.svc.cluster.local","*.long-running-qlk27-worker-svc.test-ns-88gnn.svc","*.long-running-qlk27-worker-svc.test-ns-88gnn.svc.cluster.local","*-worker-*.test-ns-88gnn.svc","*-worker-*.test-ns-88gnn.svc.cluster.local"],"ipAddresses":["10.133.0.24","10.134.0.29","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:55:52.285Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a1a2879b-852a-4d84-9837-32e2653264a4","rayCluster":"long-running-qlk27","podIPs":["10.133.0.24","10.134.0.29"]}
{"level":"info","ts":"2026-07-13T15:55:53.061Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ff844d74-53a2-4e13-8aee-775a4a660461","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:53.061Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ff844d74-53a2-4e13-8aee-775a4a660461","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:53.061Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ff844d74-53a2-4e13-8aee-775a4a660461","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:53.061Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ff844d74-53a2-4e13-8aee-775a4a660461"}
{"level":"info","ts":"2026-07-13T15:55:53.061Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"ff844d74-53a2-4e13-8aee-775a4a660461","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:55:54.263Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"944fcf73-5866-477d-9a77-809fdf79af71"}
{"level":"info","ts":"2026-07-13T15:55:54.263Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"944fcf73-5866-477d-9a77-809fdf79af71"}
{"level":"info","ts":"2026-07-13T15:55:54.263Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"944fcf73-5866-477d-9a77-809fdf79af71","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:55:54.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"944fcf73-5866-477d-9a77-809fdf79af71","Found 1 head Pod":"long-running-qlk27-head-xnt9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:55:54.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"944fcf73-5866-477d-9a77-809fdf79af71","head Pod":"long-running-qlk27-head-xnt9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qlk27-head-xnt9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:54.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"944fcf73-5866-477d-9a77-809fdf79af71","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:55:54.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"944fcf73-5866-477d-9a77-809fdf79af71","worker Pod":"long-running-qlk27-small-group-worker-tf68l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qlk27-small-group-worker-tf68l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:55:54.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"944fcf73-5866-477d-9a77-809fdf79af71","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:54.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"944fcf73-5866-477d-9a77-809fdf79af71","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:55:54.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"944fcf73-5866-477d-9a77-809fdf79af71","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:55:54.263Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"944fcf73-5866-477d-9a77-809fdf79af71","podName":"long-running-qlk27-small-group-worker-tf68l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:54.263Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"944fcf73-5866-477d-9a77-809fdf79af71","podName":"long-running-qlk27-small-group-worker-tf68l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:55:54.263Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"944fcf73-5866-477d-9a77-809fdf79af71","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:55:54.263Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"944fcf73-5866-477d-9a77-809fdf79af71","seconds":300}
{"level":"info","ts":"2026-07-13T15:55:56.062Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"fc7563e0-1a3a-40c0-9343-8d91e26e03e7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:56.062Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"fc7563e0-1a3a-40c0-9343-8d91e26e03e7","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:56.062Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"fc7563e0-1a3a-40c0-9343-8d91e26e03e7","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:56.062Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"fc7563e0-1a3a-40c0-9343-8d91e26e03e7"}
{"level":"info","ts":"2026-07-13T15:55:56.062Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"fc7563e0-1a3a-40c0-9343-8d91e26e03e7","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:55:59.062Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f29d3ca0-9e1c-4ac3-ba6d-36b5292de376","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:55:59.062Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f29d3ca0-9e1c-4ac3-ba6d-36b5292de376","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:55:59.062Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f29d3ca0-9e1c-4ac3-ba6d-36b5292de376","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:55:59.062Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f29d3ca0-9e1c-4ac3-ba6d-36b5292de376"}
{"level":"info","ts":"2026-07-13T15:55:59.062Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f29d3ca0-9e1c-4ac3-ba6d-36b5292de376","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:56:02.063Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"38e27228-6781-42ae-a884-be9323aa1349","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:56:02.063Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"38e27228-6781-42ae-a884-be9323aa1349","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:02.063Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"38e27228-6781-42ae-a884-be9323aa1349","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:56:02.063Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"38e27228-6781-42ae-a884-be9323aa1349"}
{"level":"info","ts":"2026-07-13T15:56:02.063Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"38e27228-6781-42ae-a884-be9323aa1349","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:56:03.281Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ca665291-a43a-4c4b-84da-e29101317429"}
{"level":"info","ts":"2026-07-13T15:56:03.281Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ca665291-a43a-4c4b-84da-e29101317429"}
{"level":"info","ts":"2026-07-13T15:56:03.281Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ca665291-a43a-4c4b-84da-e29101317429","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:56:03.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ca665291-a43a-4c4b-84da-e29101317429","Found 1 head Pod":"long-running-qlk27-head-xnt9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:56:03.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ca665291-a43a-4c4b-84da-e29101317429","head Pod":"long-running-qlk27-head-xnt9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qlk27-head-xnt9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:56:03.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ca665291-a43a-4c4b-84da-e29101317429","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:56:03.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ca665291-a43a-4c4b-84da-e29101317429","worker Pod":"long-running-qlk27-small-group-worker-tf68l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qlk27-small-group-worker-tf68l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:56:03.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ca665291-a43a-4c4b-84da-e29101317429","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:56:03.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ca665291-a43a-4c4b-84da-e29101317429","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:56:03.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ca665291-a43a-4c4b-84da-e29101317429","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:56:03.281Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ca665291-a43a-4c4b-84da-e29101317429","podName":"long-running-qlk27-small-group-worker-tf68l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:56:03.281Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ca665291-a43a-4c4b-84da-e29101317429","podName":"long-running-qlk27-small-group-worker-tf68l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:56:03.281Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ca665291-a43a-4c4b-84da-e29101317429","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:55:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:56:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:55:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:56:03.281Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"ca665291-a43a-4c4b-84da-e29101317429","name":"long-running-qlk27","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:55:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"long-running-qlk27-head-xnt9s","serviceName":"long-running-qlk27-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:55:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:56:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"long-running-qlk27-head-xnt9s","serviceName":"long-running-qlk27-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:56:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:55:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:56:03.294Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"00f09de6-ccd6-49e7-bc4b-5eff092b823d","rayCluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:03.294Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"b0ac0322-0dd1-476c-8d4d-abf80ee4508c","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:03.294Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"00f09de6-ccd6-49e7-bc4b-5eff092b823d","pod":"long-running-qlk27-head-xnt9s","ip":"10.133.0.24"}
{"level":"info","ts":"2026-07-13T15:56:03.294Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c993dc0a-077f-4172-8a9e-c6693ceb0cb5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:56:03.294Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c993dc0a-077f-4172-8a9e-c6693ceb0cb5","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:03.294Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c993dc0a-077f-4172-8a9e-c6693ceb0cb5","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:56:03.294Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"00f09de6-ccd6-49e7-bc4b-5eff092b823d","pod":"long-running-qlk27-small-group-worker-tf68l","ip":"10.134.0.29"}
{"level":"info","ts":"2026-07-13T15:56:03.295Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"00f09de6-ccd6-49e7-bc4b-5eff092b823d","rayCluster":"long-running-qlk27","podIPs":["10.133.0.24","10.134.0.29"]}
{"level":"info","ts":"2026-07-13T15:56:03.295Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c993dc0a-077f-4172-8a9e-c6693ceb0cb5"}
{"level":"info","ts":"2026-07-13T15:56:03.295Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c993dc0a-077f-4172-8a9e-c6693ceb0cb5","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:56:03.305Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"b0ac0322-0dd1-476c-8d4d-abf80ee4508c","name":"long-running-qlk27-head"}
{"level":"info","ts":"2026-07-13T15:56:03.315Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"b0ac0322-0dd1-476c-8d4d-abf80ee4508c","name":"long-running-qlk27-workers"}
{"level":"info","ts":"2026-07-13T15:56:03.315Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"b0ac0322-0dd1-476c-8d4d-abf80ee4508c","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:05.064Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8f7fb111-2607-4601-bbf9-98f6bbf4465d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:56:05.064Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8f7fb111-2607-4601-bbf9-98f6bbf4465d","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:05.064Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8f7fb111-2607-4601-bbf9-98f6bbf4465d","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:56:05.064Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8f7fb111-2607-4601-bbf9-98f6bbf4465d"}
{"level":"info","ts":"2026-07-13T15:56:05.064Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8f7fb111-2607-4601-bbf9-98f6bbf4465d","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:56:05.293Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"88e36520-662a-4cf5-a17f-5e7cade9810d"}
{"level":"info","ts":"2026-07-13T15:56:05.293Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"88e36520-662a-4cf5-a17f-5e7cade9810d"}
{"level":"info","ts":"2026-07-13T15:56:05.293Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"88e36520-662a-4cf5-a17f-5e7cade9810d","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:56:05.293Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"88e36520-662a-4cf5-a17f-5e7cade9810d","Found 1 head Pod":"long-running-qlk27-head-xnt9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:56:05.293Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"88e36520-662a-4cf5-a17f-5e7cade9810d","head Pod":"long-running-qlk27-head-xnt9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qlk27-head-xnt9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:56:05.293Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"88e36520-662a-4cf5-a17f-5e7cade9810d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:56:05.293Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"88e36520-662a-4cf5-a17f-5e7cade9810d","worker Pod":"long-running-qlk27-small-group-worker-tf68l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qlk27-small-group-worker-tf68l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:56:05.293Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"88e36520-662a-4cf5-a17f-5e7cade9810d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:56:05.293Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"88e36520-662a-4cf5-a17f-5e7cade9810d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:56:05.293Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"88e36520-662a-4cf5-a17f-5e7cade9810d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:56:05.293Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"88e36520-662a-4cf5-a17f-5e7cade9810d","podName":"long-running-qlk27-small-group-worker-tf68l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:56:05.293Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"88e36520-662a-4cf5-a17f-5e7cade9810d","podName":"long-running-qlk27-small-group-worker-tf68l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:56:05.293Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"88e36520-662a-4cf5-a17f-5e7cade9810d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:56:05.293Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"88e36520-662a-4cf5-a17f-5e7cade9810d","seconds":300}
{"level":"info","ts":"2026-07-13T15:56:08.065Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"850612e1-18b6-43c7-bf3d-bb0ac029ad29","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:56:08.065Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"850612e1-18b6-43c7-bf3d-bb0ac029ad29","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:08.065Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"850612e1-18b6-43c7-bf3d-bb0ac029ad29","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:56:08.065Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"850612e1-18b6-43c7-bf3d-bb0ac029ad29"}
{"level":"info","ts":"2026-07-13T15:56:08.065Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"850612e1-18b6-43c7-bf3d-bb0ac029ad29","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:56:11.066Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9f3e8758-7119-42a4-bf4c-f94ab246c94a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:56:11.066Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9f3e8758-7119-42a4-bf4c-f94ab246c94a","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:11.066Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9f3e8758-7119-42a4-bf4c-f94ab246c94a","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:56:11.066Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9f3e8758-7119-42a4-bf4c-f94ab246c94a"}
{"level":"info","ts":"2026-07-13T15:56:11.066Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9f3e8758-7119-42a4-bf4c-f94ab246c94a","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:56:12.432Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"e8ed8060-4ad2-45bc-9f5b-a60c5b52d687","rayCluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:12.433Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"e8ed8060-4ad2-45bc-9f5b-a60c5b52d687","pod":"long-running-qlk27-head-xnt9s","ip":"10.133.0.24"}
{"level":"info","ts":"2026-07-13T15:56:12.433Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"e8ed8060-4ad2-45bc-9f5b-a60c5b52d687","pod":"long-running-qlk27-small-group-worker-tf68l","ip":"10.134.0.29"}
{"level":"info","ts":"2026-07-13T15:56:12.433Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"e8ed8060-4ad2-45bc-9f5b-a60c5b52d687","rayCluster":"long-running-qlk27","podIPs":["10.133.0.24","10.134.0.29"]}
{"level":"info","ts":"2026-07-13T15:56:14.067Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8ef99dd2-c341-4544-b612-4aee37cee57e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:56:14.067Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8ef99dd2-c341-4544-b612-4aee37cee57e","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:14.067Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8ef99dd2-c341-4544-b612-4aee37cee57e","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:56:14.067Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8ef99dd2-c341-4544-b612-4aee37cee57e"}
{"level":"info","ts":"2026-07-13T15:56:14.067Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8ef99dd2-c341-4544-b612-4aee37cee57e","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:56:17.068Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"82ff8fc1-3c7d-4301-b9e7-05e1873123fd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:56:17.068Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"82ff8fc1-3c7d-4301-b9e7-05e1873123fd","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:17.068Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"82ff8fc1-3c7d-4301-b9e7-05e1873123fd","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:56:17.068Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"82ff8fc1-3c7d-4301-b9e7-05e1873123fd"}
{"level":"info","ts":"2026-07-13T15:56:17.068Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"82ff8fc1-3c7d-4301-b9e7-05e1873123fd","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:56:20.069Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a6ad32d4-9dad-4f95-b5f9-d77f43357ce5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:56:20.075Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a6ad32d4-9dad-4f95-b5f9-d77f43357ce5","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:20.075Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a6ad32d4-9dad-4f95-b5f9-d77f43357ce5","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:56:20.075Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a6ad32d4-9dad-4f95-b5f9-d77f43357ce5"}
{"level":"info","ts":"2026-07-13T15:56:20.075Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a6ad32d4-9dad-4f95-b5f9-d77f43357ce5","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:56:23.076Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"23d1d518-9b2b-4114-a054-352821c1bd94","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:56:23.076Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"23d1d518-9b2b-4114-a054-352821c1bd94","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:23.076Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"23d1d518-9b2b-4114-a054-352821c1bd94","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:56:23.076Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"23d1d518-9b2b-4114-a054-352821c1bd94"}
{"level":"info","ts":"2026-07-13T15:56:23.076Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"23d1d518-9b2b-4114-a054-352821c1bd94","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:56:26.077Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"85f432ca-ca7d-4bbf-ae0d-e40b7f51a9fa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:56:26.077Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"85f432ca-ca7d-4bbf-ae0d-e40b7f51a9fa","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:26.077Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"85f432ca-ca7d-4bbf-ae0d-e40b7f51a9fa","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:56:26.078Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"85f432ca-ca7d-4bbf-ae0d-e40b7f51a9fa"}
{"level":"info","ts":"2026-07-13T15:56:26.078Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"85f432ca-ca7d-4bbf-ae0d-e40b7f51a9fa","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:56:29.079Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"15d47225-7b27-4725-a54f-ee82705d2307","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:56:29.079Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"15d47225-7b27-4725-a54f-ee82705d2307","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:29.079Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"15d47225-7b27-4725-a54f-ee82705d2307","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:56:29.079Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"15d47225-7b27-4725-a54f-ee82705d2307"}
{"level":"info","ts":"2026-07-13T15:56:29.079Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"15d47225-7b27-4725-a54f-ee82705d2307","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:56:32.080Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7af357a9-a5f7-4e9d-98df-17148951a3d5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:56:32.080Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7af357a9-a5f7-4e9d-98df-17148951a3d5","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:32.080Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7af357a9-a5f7-4e9d-98df-17148951a3d5","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:56:32.080Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7af357a9-a5f7-4e9d-98df-17148951a3d5"}
{"level":"info","ts":"2026-07-13T15:56:32.080Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7af357a9-a5f7-4e9d-98df-17148951a3d5","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:56:34.939Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"706f9e35-f0bc-4225-8991-38d0c982ff6c","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:56:34.939Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"706f9e35-f0bc-4225-8991-38d0c982ff6c","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T15:56:34.939Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"706f9e35-f0bc-4225-8991-38d0c982ff6c","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T15:56:34.939Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"706f9e35-f0bc-4225-8991-38d0c982ff6c","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T15:56:35.081Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"173cd7b3-ad37-461c-8289-1c5a2e261243","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:56:35.081Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"173cd7b3-ad37-461c-8289-1c5a2e261243","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:35.081Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"173cd7b3-ad37-461c-8289-1c5a2e261243","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:56:35.081Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"173cd7b3-ad37-461c-8289-1c5a2e261243"}
{"level":"info","ts":"2026-07-13T15:56:35.081Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"173cd7b3-ad37-461c-8289-1c5a2e261243","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:56:38.082Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b73e7e5b-5c3c-4b30-bf86-3b96b13b7039","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:56:38.082Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b73e7e5b-5c3c-4b30-bf86-3b96b13b7039","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:38.082Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b73e7e5b-5c3c-4b30-bf86-3b96b13b7039","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:56:38.082Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b73e7e5b-5c3c-4b30-bf86-3b96b13b7039"}
{"level":"info","ts":"2026-07-13T15:56:38.082Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"b73e7e5b-5c3c-4b30-bf86-3b96b13b7039","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:56:41.083Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"48acf453-da84-442b-8f14-0ebdc1de32de","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:56:41.083Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"48acf453-da84-442b-8f14-0ebdc1de32de","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:41.083Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"48acf453-da84-442b-8f14-0ebdc1de32de","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:56:41.083Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"48acf453-da84-442b-8f14-0ebdc1de32de"}
{"level":"info","ts":"2026-07-13T15:56:41.083Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"48acf453-da84-442b-8f14-0ebdc1de32de","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:56:44.084Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"87a63326-9ff0-4a07-9551-664a6d19204c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:56:44.084Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"87a63326-9ff0-4a07-9551-664a6d19204c","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:44.084Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"87a63326-9ff0-4a07-9551-664a6d19204c","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:56:44.084Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"87a63326-9ff0-4a07-9551-664a6d19204c"}
{"level":"info","ts":"2026-07-13T15:56:44.084Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"87a63326-9ff0-4a07-9551-664a6d19204c","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:56:47.085Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"86f6b267-fc08-4258-b411-527faf55db8f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:56:47.085Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"86f6b267-fc08-4258-b411-527faf55db8f","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:47.085Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"86f6b267-fc08-4258-b411-527faf55db8f","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:56:47.085Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"86f6b267-fc08-4258-b411-527faf55db8f"}
{"level":"info","ts":"2026-07-13T15:56:47.085Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"86f6b267-fc08-4258-b411-527faf55db8f","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:56:50.086Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9813f86f-0ca0-46c8-9272-bd7b1adb9a52","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:56:50.086Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9813f86f-0ca0-46c8-9272-bd7b1adb9a52","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:50.086Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9813f86f-0ca0-46c8-9272-bd7b1adb9a52","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:56:50.086Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9813f86f-0ca0-46c8-9272-bd7b1adb9a52"}
{"level":"info","ts":"2026-07-13T15:56:50.086Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9813f86f-0ca0-46c8-9272-bd7b1adb9a52","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:56:53.087Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"11d94177-a15d-4143-aba6-1efd8eec6a89","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:56:53.087Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"11d94177-a15d-4143-aba6-1efd8eec6a89","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:53.087Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"11d94177-a15d-4143-aba6-1efd8eec6a89","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:56:53.087Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"11d94177-a15d-4143-aba6-1efd8eec6a89"}
{"level":"info","ts":"2026-07-13T15:56:53.087Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"11d94177-a15d-4143-aba6-1efd8eec6a89","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:56:56.088Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"934b5099-34fd-40d7-a651-708388e74ebb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:56:56.088Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"934b5099-34fd-40d7-a651-708388e74ebb","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:56.088Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"934b5099-34fd-40d7-a651-708388e74ebb","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:56:56.088Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"934b5099-34fd-40d7-a651-708388e74ebb"}
{"level":"info","ts":"2026-07-13T15:56:56.088Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"934b5099-34fd-40d7-a651-708388e74ebb","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:56:56.685Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2e7e6356-8367-4017-a855-2c1176dccf89"}
{"level":"info","ts":"2026-07-13T15:56:56.685Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2e7e6356-8367-4017-a855-2c1176dccf89"}
{"level":"info","ts":"2026-07-13T15:56:56.685Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2e7e6356-8367-4017-a855-2c1176dccf89","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:56:56.686Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2e7e6356-8367-4017-a855-2c1176dccf89","Found 1 head Pod":"long-running-qlk27-head-xnt9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:56:56.686Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2e7e6356-8367-4017-a855-2c1176dccf89","head Pod":"long-running-qlk27-head-xnt9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qlk27-head-xnt9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:56:56.686Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2e7e6356-8367-4017-a855-2c1176dccf89","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:56:56.686Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2e7e6356-8367-4017-a855-2c1176dccf89","worker Pod":"long-running-qlk27-small-group-worker-tf68l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qlk27-small-group-worker-tf68l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:56:56.686Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2e7e6356-8367-4017-a855-2c1176dccf89","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:56:56.686Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2e7e6356-8367-4017-a855-2c1176dccf89","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:56:56.686Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2e7e6356-8367-4017-a855-2c1176dccf89","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:56:56.686Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2e7e6356-8367-4017-a855-2c1176dccf89","podName":"long-running-qlk27-small-group-worker-tf68l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:56:56.686Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2e7e6356-8367-4017-a855-2c1176dccf89","podName":"long-running-qlk27-small-group-worker-tf68l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:56:56.686Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2e7e6356-8367-4017-a855-2c1176dccf89","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:56:56.686Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"2e7e6356-8367-4017-a855-2c1176dccf89","seconds":300}
{"level":"info","ts":"2026-07-13T15:56:57.693Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aabc01a3-a871-4daa-98b4-607160610b64"}
{"level":"info","ts":"2026-07-13T15:56:57.693Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aabc01a3-a871-4daa-98b4-607160610b64"}
{"level":"info","ts":"2026-07-13T15:56:57.693Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aabc01a3-a871-4daa-98b4-607160610b64","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:56:57.693Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aabc01a3-a871-4daa-98b4-607160610b64","Found 1 head Pod":"long-running-qlk27-head-xnt9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:56:57.693Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aabc01a3-a871-4daa-98b4-607160610b64","head Pod":"long-running-qlk27-head-xnt9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qlk27-head-xnt9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:56:57.693Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aabc01a3-a871-4daa-98b4-607160610b64","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:56:57.694Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aabc01a3-a871-4daa-98b4-607160610b64","worker Pod":"long-running-qlk27-small-group-worker-tf68l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qlk27-small-group-worker-tf68l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:56:57.694Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aabc01a3-a871-4daa-98b4-607160610b64","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:56:57.694Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aabc01a3-a871-4daa-98b4-607160610b64","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:56:57.694Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aabc01a3-a871-4daa-98b4-607160610b64","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:56:57.694Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aabc01a3-a871-4daa-98b4-607160610b64","podName":"long-running-qlk27-small-group-worker-tf68l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:56:57.694Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aabc01a3-a871-4daa-98b4-607160610b64","podName":"long-running-qlk27-small-group-worker-tf68l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:56:57.694Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aabc01a3-a871-4daa-98b4-607160610b64","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:56:57.694Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"aabc01a3-a871-4daa-98b4-607160610b64","seconds":300}
{"level":"info","ts":"2026-07-13T15:56:59.089Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"3af6e5ba-400f-4dca-9fd1-e5fecb103651","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:56:59.089Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"3af6e5ba-400f-4dca-9fd1-e5fecb103651","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:56:59.089Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"3af6e5ba-400f-4dca-9fd1-e5fecb103651","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:56:59.089Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"3af6e5ba-400f-4dca-9fd1-e5fecb103651"}
{"level":"info","ts":"2026-07-13T15:56:59.089Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"3af6e5ba-400f-4dca-9fd1-e5fecb103651","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:57:02.090Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a9ad1c1a-c4c5-4136-981e-18ef33c22140","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:02.090Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a9ad1c1a-c4c5-4136-981e-18ef33c22140","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:02.090Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a9ad1c1a-c4c5-4136-981e-18ef33c22140","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:02.090Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a9ad1c1a-c4c5-4136-981e-18ef33c22140"}
{"level":"info","ts":"2026-07-13T15:57:02.090Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"a9ad1c1a-c4c5-4136-981e-18ef33c22140","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:57:05.091Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"0b8d0e12-92b0-4a5f-b3be-09a6ada8bd66","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:05.091Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"0b8d0e12-92b0-4a5f-b3be-09a6ada8bd66","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:05.091Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"0b8d0e12-92b0-4a5f-b3be-09a6ada8bd66","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:05.091Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"0b8d0e12-92b0-4a5f-b3be-09a6ada8bd66"}
{"level":"info","ts":"2026-07-13T15:57:05.091Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"0b8d0e12-92b0-4a5f-b3be-09a6ada8bd66","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:57:06.731Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a6af4158-dd8e-41b4-9b13-64ac2b342e4f"}
{"level":"info","ts":"2026-07-13T15:57:06.731Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a6af4158-dd8e-41b4-9b13-64ac2b342e4f"}
{"level":"info","ts":"2026-07-13T15:57:06.732Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a6af4158-dd8e-41b4-9b13-64ac2b342e4f","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:06.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a6af4158-dd8e-41b4-9b13-64ac2b342e4f","Found 1 head Pod":"long-running-qlk27-head-xnt9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:57:06.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a6af4158-dd8e-41b4-9b13-64ac2b342e4f","head Pod":"long-running-qlk27-head-xnt9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qlk27-head-xnt9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:06.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a6af4158-dd8e-41b4-9b13-64ac2b342e4f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:57:06.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a6af4158-dd8e-41b4-9b13-64ac2b342e4f","worker Pod":"long-running-qlk27-small-group-worker-tf68l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qlk27-small-group-worker-tf68l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:06.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a6af4158-dd8e-41b4-9b13-64ac2b342e4f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:06.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a6af4158-dd8e-41b4-9b13-64ac2b342e4f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:57:06.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a6af4158-dd8e-41b4-9b13-64ac2b342e4f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:06.732Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a6af4158-dd8e-41b4-9b13-64ac2b342e4f","podName":"long-running-qlk27-small-group-worker-tf68l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:06.732Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a6af4158-dd8e-41b4-9b13-64ac2b342e4f","podName":"long-running-qlk27-small-group-worker-tf68l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:06.732Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a6af4158-dd8e-41b4-9b13-64ac2b342e4f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:57:06.732Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"a6af4158-dd8e-41b4-9b13-64ac2b342e4f","seconds":300}
{"level":"info","ts":"2026-07-13T15:57:07.741Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"f2f46b68-05d7-4cae-aebd-0b351cf794a1"}
{"level":"info","ts":"2026-07-13T15:57:07.741Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"f2f46b68-05d7-4cae-aebd-0b351cf794a1"}
{"level":"info","ts":"2026-07-13T15:57:07.741Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"f2f46b68-05d7-4cae-aebd-0b351cf794a1","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:07.741Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"f2f46b68-05d7-4cae-aebd-0b351cf794a1","Found 1 head Pod":"long-running-qlk27-head-xnt9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:57:07.741Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"f2f46b68-05d7-4cae-aebd-0b351cf794a1","head Pod":"long-running-qlk27-head-xnt9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qlk27-head-xnt9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:07.741Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"f2f46b68-05d7-4cae-aebd-0b351cf794a1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:57:07.741Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"f2f46b68-05d7-4cae-aebd-0b351cf794a1","worker Pod":"long-running-qlk27-small-group-worker-tf68l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qlk27-small-group-worker-tf68l. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:07.741Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"f2f46b68-05d7-4cae-aebd-0b351cf794a1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:07.741Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"f2f46b68-05d7-4cae-aebd-0b351cf794a1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:57:07.741Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"f2f46b68-05d7-4cae-aebd-0b351cf794a1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:07.741Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"f2f46b68-05d7-4cae-aebd-0b351cf794a1","podName":"long-running-qlk27-small-group-worker-tf68l","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:55:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:57:07.741Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"f2f46b68-05d7-4cae-aebd-0b351cf794a1","podName":"long-running-qlk27-small-group-worker-tf68l","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:55:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:57:07.741Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"f2f46b68-05d7-4cae-aebd-0b351cf794a1","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:57:07.741Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"f2f46b68-05d7-4cae-aebd-0b351cf794a1","name":"long-running-qlk27","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:56:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"long-running-qlk27-head-xnt9s","serviceName":"long-running-qlk27-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:56:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:55:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:57:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"long-running-qlk27-head-xnt9s","serviceName":"long-running-qlk27-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:56:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:55:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:57:07.753Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"b46996c4-2c82-4282-a1a7-5ca8ad1d1e5f","rayCluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:07.753Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"5bb9956d-09a9-43f8-8fd0-04c5d2dc89bf","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:07.753Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9cd92821-a368-4c19-9398-610cef758e2c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:07.754Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9cd92821-a368-4c19-9398-610cef758e2c","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:07.754Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"b46996c4-2c82-4282-a1a7-5ca8ad1d1e5f","pod":"long-running-qlk27-head-xnt9s","ip":"10.133.0.24"}
{"level":"info","ts":"2026-07-13T15:57:07.754Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"b46996c4-2c82-4282-a1a7-5ca8ad1d1e5f","pod":"long-running-qlk27-small-group-worker-tf68l","ip":"10.134.0.29"}
{"level":"info","ts":"2026-07-13T15:57:07.754Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9cd92821-a368-4c19-9398-610cef758e2c","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:07.754Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"b46996c4-2c82-4282-a1a7-5ca8ad1d1e5f","rayCluster":"long-running-qlk27","podIPs":["10.133.0.24","10.134.0.29"]}
{"level":"info","ts":"2026-07-13T15:57:07.754Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9cd92821-a368-4c19-9398-610cef758e2c"}
{"level":"info","ts":"2026-07-13T15:57:07.754Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9cd92821-a368-4c19-9398-610cef758e2c","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:57:07.770Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"5bb9956d-09a9-43f8-8fd0-04c5d2dc89bf","name":"long-running-qlk27-head"}
{"level":"info","ts":"2026-07-13T15:57:07.779Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"5bb9956d-09a9-43f8-8fd0-04c5d2dc89bf","name":"long-running-qlk27-workers"}
{"level":"info","ts":"2026-07-13T15:57:07.779Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"5bb9956d-09a9-43f8-8fd0-04c5d2dc89bf","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:08.092Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9f2a0fdc-5414-43cb-9767-fd0505bfc56f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:08.092Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9f2a0fdc-5414-43cb-9767-fd0505bfc56f","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:08.092Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9f2a0fdc-5414-43cb-9767-fd0505bfc56f","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:08.092Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9f2a0fdc-5414-43cb-9767-fd0505bfc56f"}
{"level":"info","ts":"2026-07-13T15:57:08.092Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"9f2a0fdc-5414-43cb-9767-fd0505bfc56f","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:57:09.754Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"34d6c76d-f03c-43f0-8a7b-77c83678b61a"}
{"level":"info","ts":"2026-07-13T15:57:09.754Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"34d6c76d-f03c-43f0-8a7b-77c83678b61a"}
{"level":"info","ts":"2026-07-13T15:57:09.754Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"34d6c76d-f03c-43f0-8a7b-77c83678b61a","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:09.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"34d6c76d-f03c-43f0-8a7b-77c83678b61a","Found 1 head Pod":"long-running-qlk27-head-xnt9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:57:09.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"34d6c76d-f03c-43f0-8a7b-77c83678b61a","head Pod":"long-running-qlk27-head-xnt9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qlk27-head-xnt9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:09.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"34d6c76d-f03c-43f0-8a7b-77c83678b61a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:57:09.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"34d6c76d-f03c-43f0-8a7b-77c83678b61a","worker Pod":"long-running-qlk27-small-group-worker-tf68l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qlk27-small-group-worker-tf68l. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:09.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"34d6c76d-f03c-43f0-8a7b-77c83678b61a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:09.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"34d6c76d-f03c-43f0-8a7b-77c83678b61a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:57:09.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"34d6c76d-f03c-43f0-8a7b-77c83678b61a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:09.754Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"34d6c76d-f03c-43f0-8a7b-77c83678b61a","podName":"long-running-qlk27-small-group-worker-tf68l","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:55:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:57:09.754Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"34d6c76d-f03c-43f0-8a7b-77c83678b61a","podName":"long-running-qlk27-small-group-worker-tf68l","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:55:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:57:09.754Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"34d6c76d-f03c-43f0-8a7b-77c83678b61a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:57:09.754Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"34d6c76d-f03c-43f0-8a7b-77c83678b61a","seconds":300}
{"level":"info","ts":"2026-07-13T15:57:11.093Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"fce7625c-c91b-486c-a192-469dcb439dce","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:11.093Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"fce7625c-c91b-486c-a192-469dcb439dce","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:11.093Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"fce7625c-c91b-486c-a192-469dcb439dce","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:11.093Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"fce7625c-c91b-486c-a192-469dcb439dce"}
{"level":"info","ts":"2026-07-13T15:57:11.093Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"fce7625c-c91b-486c-a192-469dcb439dce","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:57:12.433Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"8ea488d7-a46e-4297-a8d4-920248b4cc77","rayCluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:12.433Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"8ea488d7-a46e-4297-a8d4-920248b4cc77","pod":"long-running-qlk27-head-xnt9s","ip":"10.133.0.24"}
{"level":"info","ts":"2026-07-13T15:57:12.433Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"8ea488d7-a46e-4297-a8d4-920248b4cc77","pod":"long-running-qlk27-small-group-worker-tf68l","ip":"10.134.0.29"}
{"level":"info","ts":"2026-07-13T15:57:12.433Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"8ea488d7-a46e-4297-a8d4-920248b4cc77","rayCluster":"long-running-qlk27","podIPs":["10.133.0.24","10.134.0.29"]}
{"level":"info","ts":"2026-07-13T15:57:14.094Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c61d302f-4986-4667-a505-ab45c0c7a6f7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:14.094Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c61d302f-4986-4667-a505-ab45c0c7a6f7","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:14.094Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c61d302f-4986-4667-a505-ab45c0c7a6f7","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:14.095Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c61d302f-4986-4667-a505-ab45c0c7a6f7"}
{"level":"info","ts":"2026-07-13T15:57:14.095Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"c61d302f-4986-4667-a505-ab45c0c7a6f7","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:57:17.095Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"74480c61-3bb9-4562-b361-cc6f2e798439","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:17.095Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"74480c61-3bb9-4562-b361-cc6f2e798439","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:17.095Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"74480c61-3bb9-4562-b361-cc6f2e798439","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:17.096Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"74480c61-3bb9-4562-b361-cc6f2e798439"}
{"level":"info","ts":"2026-07-13T15:57:17.096Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"74480c61-3bb9-4562-b361-cc6f2e798439","RayCluster":"long-running-qlk27","State":""}
{"level":"info","ts":"2026-07-13T15:57:18.762Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"af007807-6aee-4b8d-97f0-73cebcdb0f8a"}
{"level":"info","ts":"2026-07-13T15:57:18.762Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"af007807-6aee-4b8d-97f0-73cebcdb0f8a"}
{"level":"info","ts":"2026-07-13T15:57:18.762Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"af007807-6aee-4b8d-97f0-73cebcdb0f8a","1 head service found":"long-running-qlk27-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:18.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"af007807-6aee-4b8d-97f0-73cebcdb0f8a","Found 1 head Pod":"long-running-qlk27-head-xnt9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:57:18.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"af007807-6aee-4b8d-97f0-73cebcdb0f8a","head Pod":"long-running-qlk27-head-xnt9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qlk27-head-xnt9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:18.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"af007807-6aee-4b8d-97f0-73cebcdb0f8a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:57:18.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"af007807-6aee-4b8d-97f0-73cebcdb0f8a","worker Pod":"long-running-qlk27-small-group-worker-tf68l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qlk27-small-group-worker-tf68l. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:18.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"af007807-6aee-4b8d-97f0-73cebcdb0f8a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:18.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"af007807-6aee-4b8d-97f0-73cebcdb0f8a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:57:18.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"af007807-6aee-4b8d-97f0-73cebcdb0f8a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:18.762Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"af007807-6aee-4b8d-97f0-73cebcdb0f8a","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-13T15:57:18.762Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"af007807-6aee-4b8d-97f0-73cebcdb0f8a","name":"long-running-qlk27","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:57:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"long-running-qlk27-head-xnt9s","serviceName":"long-running-qlk27-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:56:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:55:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:57:18Z","stateTransitionTimes":{"ready":"2026-07-13T15:57:18Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"long-running-qlk27-head-xnt9s","serviceName":"long-running-qlk27-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:56:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:57:18Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:55:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:55:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:57:18.775Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"2944eb12-2ed9-49bc-9ac8-df01da492916","rayCluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:18.775Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"45f2fb16-719b-407f-8c06-1440d32a795d","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:18.775Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"2944eb12-2ed9-49bc-9ac8-df01da492916","pod":"long-running-qlk27-head-xnt9s","ip":"10.133.0.24"}
{"level":"info","ts":"2026-07-13T15:57:18.775Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"2944eb12-2ed9-49bc-9ac8-df01da492916","pod":"long-running-qlk27-small-group-worker-tf68l","ip":"10.134.0.29"}
{"level":"info","ts":"2026-07-13T15:57:18.775Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"2944eb12-2ed9-49bc-9ac8-df01da492916","rayCluster":"long-running-qlk27","podIPs":["10.133.0.24","10.134.0.29"]}
{"level":"info","ts":"2026-07-13T15:57:18.775Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2420a2b5-1511-4d28-9692-0857c9f8baec","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:18.775Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2420a2b5-1511-4d28-9692-0857c9f8baec","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:18.775Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2420a2b5-1511-4d28-9692-0857c9f8baec","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:18.775Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2420a2b5-1511-4d28-9692-0857c9f8baec"}
{"level":"info","ts":"2026-07-13T15:57:18.775Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2420a2b5-1511-4d28-9692-0857c9f8baec","head service name":"long-running-qlk27-head-svc","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:57:18.775Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2420a2b5-1511-4d28-9692-0857c9f8baec","head service URL":"long-running-qlk27-head-svc.test-ns-88gnn.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-13T15:57:18.775Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2420a2b5-1511-4d28-9692-0857c9f8baec"}
{"level":"info","ts":"2026-07-13T15:57:18.775Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2420a2b5-1511-4d28-9692-0857c9f8baec","command":["if","!","ray","job","status","--address","http://long-running-qlk27-head-svc.test-ns-88gnn.svc.cluster.local:8265","long-running-tsqx8",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://long-running-qlk27-head-svc.test-ns-88gnn.svc.cluster.local:8265","--no-wait","--submission-id","long-running-tsqx8","--","python /home/ray/jobs/long_running.py",";","fi",";","ray","job","logs","--address","http://long-running-qlk27-head-svc.test-ns-88gnn.svc.cluster.local:8265","--follow","long-running-tsqx8"]}
{"level":"info","ts":"2026-07-13T15:57:18.788Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2420a2b5-1511-4d28-9692-0857c9f8baec","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-07-13T15:57:18.788Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2420a2b5-1511-4d28-9692-0857c9f8baec","SubmissionMode":"K8sJobMode","RayCluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:18.788Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2420a2b5-1511-4d28-9692-0857c9f8baec","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-tsqx8","rayClusterName":"long-running-qlk27","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:55:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-tsqx8","rayClusterName":"long-running-qlk27","dashboardURL":"long-running-qlk27-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T15:55:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:57:18.788Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"2420a2b5-1511-4d28-9692-0857c9f8baec","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T15:57:18.793Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"45f2fb16-719b-407f-8c06-1440d32a795d","name":"long-running-qlk27-head"}
{"level":"info","ts":"2026-07-13T15:57:18.799Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8fab3637-f846-4a99-84df-2157f44635ba","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:18.799Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8fab3637-f846-4a99-84df-2157f44635ba","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:18.799Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8fab3637-f846-4a99-84df-2157f44635ba","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:18.799Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8fab3637-f846-4a99-84df-2157f44635ba"}
{"level":"info","ts":"2026-07-13T15:57:18.799Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8fab3637-f846-4a99-84df-2157f44635ba","head service name":"long-running-qlk27-head-svc","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:57:18.799Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8fab3637-f846-4a99-84df-2157f44635ba","head service URL":"long-running-qlk27-head-svc.test-ns-88gnn.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-13T15:57:18.799Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8fab3637-f846-4a99-84df-2157f44635ba","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-07-13T15:57:18.799Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8fab3637-f846-4a99-84df-2157f44635ba","SubmissionMode":"K8sJobMode","RayCluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:18.799Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8fab3637-f846-4a99-84df-2157f44635ba","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-tsqx8","rayClusterName":"long-running-qlk27","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:55:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-tsqx8","rayClusterName":"long-running-qlk27","dashboardURL":"long-running-qlk27-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T15:55:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:57:18.799Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8fab3637-f846-4a99-84df-2157f44635ba","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T15:57:18.807Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8fab3637-f846-4a99-84df-2157f44635ba","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-13T15:57:18.807Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8fab3637-f846-4a99-84df-2157f44635ba"}
{"level":"error","ts":"2026-07-13T15:57:18.807Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8fab3637-f846-4a99-84df-2157f44635ba","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:18.808Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"06892051-7c4e-464e-9225-1f4b95792151","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:18.808Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"06892051-7c4e-464e-9225-1f4b95792151","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:18.808Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"06892051-7c4e-464e-9225-1f4b95792151","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:18.808Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"06892051-7c4e-464e-9225-1f4b95792151"}
{"level":"info","ts":"2026-07-13T15:57:18.813Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"45f2fb16-719b-407f-8c06-1440d32a795d","name":"long-running-qlk27-workers"}
{"level":"info","ts":"2026-07-13T15:57:18.813Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"45f2fb16-719b-407f-8c06-1440d32a795d","cluster":"long-running-qlk27"}
{"level":"error","ts":"2026-07-13T15:57:18.816Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"06892051-7c4e-464e-9225-1f4b95792151","JobId":"long-running-tsqx8","error":"Job long-running-tsqx8 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:18.816Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"06892051-7c4e-464e-9225-1f4b95792151"}
{"level":"error","ts":"2026-07-13T15:57:18.816Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"06892051-7c4e-464e-9225-1f4b95792151","error":"Job long-running-tsqx8 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:18.816Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"29f7247b-29b4-4bda-bf96-8df3ed983aaa","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:18.816Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"29f7247b-29b4-4bda-bf96-8df3ed983aaa","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:18.816Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"29f7247b-29b4-4bda-bf96-8df3ed983aaa","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:18.816Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"29f7247b-29b4-4bda-bf96-8df3ed983aaa"}
{"level":"error","ts":"2026-07-13T15:57:18.826Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"29f7247b-29b4-4bda-bf96-8df3ed983aaa","JobId":"long-running-tsqx8","error":"Job long-running-tsqx8 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:18.826Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"29f7247b-29b4-4bda-bf96-8df3ed983aaa"}
{"level":"error","ts":"2026-07-13T15:57:18.826Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"29f7247b-29b4-4bda-bf96-8df3ed983aaa","error":"Job long-running-tsqx8 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:18.826Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"41b02a03-2e8a-4c2d-b9e1-de7d87d72b96","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:18.826Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"41b02a03-2e8a-4c2d-b9e1-de7d87d72b96","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:18.826Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"41b02a03-2e8a-4c2d-b9e1-de7d87d72b96","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:18.827Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"41b02a03-2e8a-4c2d-b9e1-de7d87d72b96"}
{"level":"error","ts":"2026-07-13T15:57:18.831Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"41b02a03-2e8a-4c2d-b9e1-de7d87d72b96","JobId":"long-running-tsqx8","error":"Job long-running-tsqx8 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:18.831Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"41b02a03-2e8a-4c2d-b9e1-de7d87d72b96"}
{"level":"error","ts":"2026-07-13T15:57:18.831Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"41b02a03-2e8a-4c2d-b9e1-de7d87d72b96","error":"Job long-running-tsqx8 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:18.847Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d86da4db-176d-49ae-88cd-9caad5b4149d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:18.847Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d86da4db-176d-49ae-88cd-9caad5b4149d","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:18.847Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d86da4db-176d-49ae-88cd-9caad5b4149d","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:18.847Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d86da4db-176d-49ae-88cd-9caad5b4149d"}
{"level":"error","ts":"2026-07-13T15:57:18.851Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d86da4db-176d-49ae-88cd-9caad5b4149d","JobId":"long-running-tsqx8","error":"Job long-running-tsqx8 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:18.851Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d86da4db-176d-49ae-88cd-9caad5b4149d"}
{"level":"error","ts":"2026-07-13T15:57:18.851Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"d86da4db-176d-49ae-88cd-9caad5b4149d","error":"Job long-running-tsqx8 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:18.932Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f2b2c379-dff9-4db0-b579-2536bd684b7a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:18.932Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f2b2c379-dff9-4db0-b579-2536bd684b7a","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:18.932Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f2b2c379-dff9-4db0-b579-2536bd684b7a","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:18.932Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f2b2c379-dff9-4db0-b579-2536bd684b7a"}
{"level":"error","ts":"2026-07-13T15:57:18.936Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f2b2c379-dff9-4db0-b579-2536bd684b7a","JobId":"long-running-tsqx8","error":"Job long-running-tsqx8 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:18.936Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f2b2c379-dff9-4db0-b579-2536bd684b7a"}
{"level":"error","ts":"2026-07-13T15:57:18.936Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"f2b2c379-dff9-4db0-b579-2536bd684b7a","error":"Job long-running-tsqx8 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:19.096Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7fe022fa-d423-434b-a7a3-11d63b2d9315","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:19.096Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7fe022fa-d423-434b-a7a3-11d63b2d9315","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:19.096Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7fe022fa-d423-434b-a7a3-11d63b2d9315","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:19.096Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7fe022fa-d423-434b-a7a3-11d63b2d9315"}
{"level":"error","ts":"2026-07-13T15:57:19.101Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7fe022fa-d423-434b-a7a3-11d63b2d9315","JobId":"long-running-tsqx8","error":"Job long-running-tsqx8 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:19.101Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7fe022fa-d423-434b-a7a3-11d63b2d9315"}
{"level":"error","ts":"2026-07-13T15:57:19.101Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"7fe022fa-d423-434b-a7a3-11d63b2d9315","error":"Job long-running-tsqx8 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:19.303Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"68399b27-c205-4c99-ad10-69507c96ddb8","DeletionTimestamp":"2026-07-13 15:57:19 +0000 UTC"}
{"level":"info","ts":"2026-07-13T15:57:19.303Z","logger":"controllers.RayJob","msg":"Stop a ray job","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"68399b27-c205-4c99-ad10-69507c96ddb8","rayJob":"long-running-tsqx8"}
{"level":"error","ts":"2026-07-13T15:57:19.308Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"68399b27-c205-4c99-ad10-69507c96ddb8","error":"invalid character 'J' looking for beginning of value","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:123\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:19.308Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"68399b27-c205-4c99-ad10-69507c96ddb8","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T15:57:19.347Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"5cbec253-a807-4172-8e7a-c39fd23d866c","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:19.347Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"5cbec253-a807-4172-8e7a-c39fd23d866c","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T15:57:19.363Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"5cbec253-a807-4172-8e7a-c39fd23d866c"}
{"level":"info","ts":"2026-07-13T15:57:19.363Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"5cbec253-a807-4172-8e7a-c39fd23d866c","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-13T15:57:19.363Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"5cbec253-a807-4172-8e7a-c39fd23d866c","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-jcjlq","rayClusterName":"counter-kwrfd","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:57:19Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:57:19.363Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"5cbec253-a807-4172-8e7a-c39fd23d866c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T15:57:19.385Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"460753de-eb7a-4f15-af7f-575be4c04fca","namespacedName":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:19.385Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"460753de-eb7a-4f15-af7f-575be4c04fca","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:19.385Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"460753de-eb7a-4f15-af7f-575be4c04fca","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:19.385Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"40e9bfcb-b9b7-4926-bb82-005519aa626f","rayCluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:19.385Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"31b984d1-386d-4489-bbff-52c9c635b10c"}
{"level":"info","ts":"2026-07-13T15:57:19.385Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"reconcileID":"4a8c4d77-6761-41cc-9cbe-a69a9f8f23cb"}
{"level":"info","ts":"2026-07-13T15:57:19.385Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"460753de-eb7a-4f15-af7f-575be4c04fca","namespace":"test-ns-88gnn","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-13T15:57:19.385Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"40e9bfcb-b9b7-4926-bb82-005519aa626f","pod":"long-running-qlk27-small-group-worker-tf68l","ip":"10.134.0.29"}
{"level":"info","ts":"2026-07-13T15:57:19.385Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"40e9bfcb-b9b7-4926-bb82-005519aa626f","pod":"long-running-qlk27-head-xnt9s","ip":"10.133.0.24"}
{"level":"info","ts":"2026-07-13T15:57:19.385Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"40e9bfcb-b9b7-4926-bb82-005519aa626f","rayCluster":"long-running-qlk27","podIPs":["10.134.0.29","10.133.0.24"]}
{"level":"info","ts":"2026-07-13T15:57:19.387Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"4c2f4fc2-d726-4a69-80d3-65ac388502e7"}
{"level":"info","ts":"2026-07-13T15:57:19.387Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1c482cee-cedb-4abf-a76e-1c8ce72cbab2","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:19.387Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1c482cee-cedb-4abf-a76e-1c8ce72cbab2"}
{"level":"info","ts":"2026-07-13T15:57:19.387Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1c482cee-cedb-4abf-a76e-1c8ce72cbab2","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-13T15:57:19.387Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1c482cee-cedb-4abf-a76e-1c8ce72cbab2","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-85wgx","rayClusterName":"counter-rpbk5","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:57:19Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:57:19.387Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1c482cee-cedb-4abf-a76e-1c8ce72cbab2","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T15:57:19.391Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"460753de-eb7a-4f15-af7f-575be4c04fca","name":"kuberay-gateway-access","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:57:19.396Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1c482cee-cedb-4abf-a76e-1c8ce72cbab2","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-13T15:57:19.396Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1c482cee-cedb-4abf-a76e-1c8ce72cbab2"}
{"level":"error","ts":"2026-07-13T15:57:19.396Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1c482cee-cedb-4abf-a76e-1c8ce72cbab2","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:19.396Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f9d5d5ed-0210-4d46-8c10-7ac15c7eefa8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:19.396Z","logger":"controllers.RayJob","msg":"Try to transition the status to `Suspending`","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f9d5d5ed-0210-4d46-8c10-7ac15c7eefa8","oldStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T15:57:19.396Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f9d5d5ed-0210-4d46-8c10-7ac15c7eefa8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-jcjlq","rayClusterName":"counter-kwrfd","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:57:19Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-jcjlq","rayClusterName":"counter-kwrfd","jobDeploymentStatus":"Suspending","startTime":"2026-07-13T15:57:19Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:57:19.396Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f9d5d5ed-0210-4d46-8c10-7ac15c7eefa8","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Suspending"}
{"level":"info","ts":"2026-07-13T15:57:19.400Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"460753de-eb7a-4f15-af7f-575be4c04fca","name":"test-ns-88gnn-long-running-qlk27","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-13T15:57:19.421Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"6370b970-7ca3-4145-9377-e3c4ed1dd842"}
{"level":"info","ts":"2026-07-13T15:57:19.422Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"2a710f51-ffe0-48af-bc34-04d46da0f33c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:19.422Z","logger":"controllers.RayJob","msg":"Try to transition the status to `Suspending`","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"2a710f51-ffe0-48af-bc34-04d46da0f33c","oldStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T15:57:19.422Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"2a710f51-ffe0-48af-bc34-04d46da0f33c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-jcjlq","rayClusterName":"counter-kwrfd","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:57:19Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-jcjlq","rayClusterName":"counter-kwrfd","jobDeploymentStatus":"Suspending","startTime":"2026-07-13T15:57:19Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:57:19.422Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"2a710f51-ffe0-48af-bc34-04d46da0f33c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Suspending"}
{"level":"info","ts":"2026-07-13T15:57:19.428Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"460753de-eb7a-4f15-af7f-575be4c04fca","configMap":"kube-rbac-proxy-config-long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:19.429Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"2a710f51-ffe0-48af-bc34-04d46da0f33c","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-13T15:57:19.429Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"2a710f51-ffe0-48af-bc34-04d46da0f33c"}
{"level":"error","ts":"2026-07-13T15:57:19.429Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"2a710f51-ffe0-48af-bc34-04d46da0f33c","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:19.429Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a40e3f05-8a21-405f-bb09-002c46943fad","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:19.429Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a40e3f05-8a21-405f-bb09-002c46943fad","RayCluster":{"name":"counter-kwrfd","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:19.429Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a40e3f05-8a21-405f-bb09-002c46943fad","isClusterDeleted":true}
{"level":"info","ts":"2026-07-13T15:57:19.429Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a40e3f05-8a21-405f-bb09-002c46943fad","Kubernetes Job":""}
{"level":"info","ts":"2026-07-13T15:57:19.429Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a40e3f05-8a21-405f-bb09-002c46943fad","isJobDeleted":true}
{"level":"info","ts":"2026-07-13T15:57:19.429Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a40e3f05-8a21-405f-bb09-002c46943fad","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-jcjlq","rayClusterName":"counter-kwrfd","jobDeploymentStatus":"Suspending","startTime":"2026-07-13T15:57:19Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-07-13T15:57:19Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:57:19.429Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a40e3f05-8a21-405f-bb09-002c46943fad","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspending","new JobDeploymentStatus":"Suspended"}
{"level":"info","ts":"2026-07-13T15:57:19.432Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"460753de-eb7a-4f15-af7f-575be4c04fca","serviceAccount":"long-running-qlk27-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:57:19.440Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ed34fb46-0102-4a6e-9db5-5ee48c90c097","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:19.440Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ed34fb46-0102-4a6e-9db5-5ee48c90c097","RayCluster":{"name":"counter-kwrfd","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:19.440Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ed34fb46-0102-4a6e-9db5-5ee48c90c097","isClusterDeleted":true}
{"level":"info","ts":"2026-07-13T15:57:19.440Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ed34fb46-0102-4a6e-9db5-5ee48c90c097","Kubernetes Job":""}
{"level":"info","ts":"2026-07-13T15:57:19.440Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ed34fb46-0102-4a6e-9db5-5ee48c90c097","isJobDeleted":true}
{"level":"info","ts":"2026-07-13T15:57:19.440Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ed34fb46-0102-4a6e-9db5-5ee48c90c097","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-jcjlq","rayClusterName":"counter-kwrfd","jobDeploymentStatus":"Suspending","startTime":"2026-07-13T15:57:19Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-07-13T15:57:19Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:57:19.440Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ed34fb46-0102-4a6e-9db5-5ee48c90c097","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspending","new JobDeploymentStatus":"Suspended"}
{"level":"info","ts":"2026-07-13T15:57:19.521Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ed34fb46-0102-4a6e-9db5-5ee48c90c097","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-13T15:57:19.521Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ed34fb46-0102-4a6e-9db5-5ee48c90c097"}
{"level":"error","ts":"2026-07-13T15:57:19.521Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ed34fb46-0102-4a6e-9db5-5ee48c90c097","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:19.522Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"85414d93-52e4-4661-b881-b17592800c4c","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:19.621Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"0368d080-ecae-490b-8281-cff88a1d04e5","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:19.627Z","logger":"raycluster-resource","msg":"default","name":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:19.627Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-qlk27","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:57:19.732Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"460753de-eb7a-4f15-af7f-575be4c04fca","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:19.732Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a6f2a603-19f7-4e16-9ebe-b24e2a6a10a9","namespacedName":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:19.732Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a6f2a603-19f7-4e16-9ebe-b24e2a6a10a9","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:19.732Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a6f2a603-19f7-4e16-9ebe-b24e2a6a10a9","cluster":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:19.732Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a6f2a603-19f7-4e16-9ebe-b24e2a6a10a9","namespace":"test-ns-88gnn","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-13T15:57:19.733Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a6f2a603-19f7-4e16-9ebe-b24e2a6a10a9","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-13T15:57:19.733Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"965de6c9-41fe-484d-9b9e-0337f190efc6"}
{"level":"info","ts":"2026-07-13T15:57:19.733Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"0f0cc6ae-67dd-4bd3-9dff-10e3a7e6b41f","namespace":"test-ns-88gnn","clusterName":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:19.733Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-88gnn"},"reconcileID":"8dddf1fe-d1b5-408f-8785-aafe7ac6cdb0"}
{"level":"error","ts":"2026-07-13T15:57:19.736Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a6f2a603-19f7-4e16-9ebe-b24e2a6a10a9","error":"rayclusters.ray.io \"long-running-qlk27\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:19.736Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a6f2a603-19f7-4e16-9ebe-b24e2a6a10a9"}
{"level":"error","ts":"2026-07-13T15:57:19.736Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a6f2a603-19f7-4e16-9ebe-b24e2a6a10a9","error":"rayclusters.ray.io \"long-running-qlk27\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:19.736Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"c71ada89-2a41-4ff0-8441-83109e376aa0","namespacedName":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:19.736Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"c71ada89-2a41-4ff0-8441-83109e376aa0"}
{"level":"info","ts":"2026-07-13T15:57:19.736Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"c71ada89-2a41-4ff0-8441-83109e376aa0","namespace":"test-ns-88gnn","authClusters":0}
{"level":"info","ts":"2026-07-13T15:57:19.736Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"c71ada89-2a41-4ff0-8441-83109e376aa0","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:57:19.741Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"4cc31f25-1ce9-4e50-adca-fda32171522d","namespacedName":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:19.741Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"4cc31f25-1ce9-4e50-adca-fda32171522d"}
{"level":"info","ts":"2026-07-13T15:57:19.741Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"4cc31f25-1ce9-4e50-adca-fda32171522d","namespace":"test-ns-88gnn","authClusters":0}
{"level":"info","ts":"2026-07-13T15:57:19.741Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"4cc31f25-1ce9-4e50-adca-fda32171522d","namespace":"test-ns-88gnn"}
{"level":"error","ts":"2026-07-13T15:57:19.741Z","msg":"Failed to delete CA issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"0f0cc6ae-67dd-4bd3-9dff-10e3a7e6b41f","issuer":"ray-ca-issuer-long-running-qlk27","error":"issuers.cert-manager.io \"ray-ca-issuer-long-running-qlk27\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:247\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:57:19.744Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"0f0cc6ae-67dd-4bd3-9dff-10e3a7e6b41f","certificate":"ray-head-cert-long-running-qlk27","error":"certificates.cert-manager.io \"ray-head-cert-long-running-qlk27\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:19.750Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"84b2ac15-adde-4acf-9d67-63f395f9f644"}
{"level":"info","ts":"2026-07-13T15:57:19.750Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"8c3a6a75-4416-423c-a857-4d47cbbbbf41"}
{"level":"info","ts":"2026-07-13T15:57:19.751Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a083b8c5-dd6e-4fc0-9c5d-21e700ed1781","namespacedName":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:19.751Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a083b8c5-dd6e-4fc0-9c5d-21e700ed1781"}
{"level":"info","ts":"2026-07-13T15:57:19.751Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a083b8c5-dd6e-4fc0-9c5d-21e700ed1781","namespace":"test-ns-88gnn","authClusters":0}
{"level":"info","ts":"2026-07-13T15:57:19.751Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"a083b8c5-dd6e-4fc0-9c5d-21e700ed1781","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:57:19.769Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"0f0cc6ae-67dd-4bd3-9dff-10e3a7e6b41f","errorCount":2}
{"level":"info","ts":"2026-07-13T15:57:19.769Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"0f0cc6ae-67dd-4bd3-9dff-10e3a7e6b41f"}
{"level":"error","ts":"2026-07-13T15:57:19.769Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"0f0cc6ae-67dd-4bd3-9dff-10e3a7e6b41f","error":"issuers.cert-manager.io \"ray-ca-issuer-long-running-qlk27\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:19.775Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"8e82cf59-7278-40e3-9f10-7040963b8b37","namespace":"test-ns-88gnn","clusterName":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:19.775Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qlk27","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"long-running-qlk27","reconcileID":"8e82cf59-7278-40e3-9f10-7040963b8b37","namespace":"test-ns-88gnn","clusterName":"long-running-qlk27"}
{"level":"info","ts":"2026-07-13T15:57:20.346Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"2b10b28d-fe07-4f98-a503-635c43f6abc7","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:20.346Z","logger":"controllers.RayJob","msg":"The status is 'Suspended', but the suspend flag is false. Transition the status to 'New'.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"2b10b28d-fe07-4f98-a503-635c43f6abc7"}
{"level":"info","ts":"2026-07-13T15:57:20.346Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"2b10b28d-fe07-4f98-a503-635c43f6abc7","oldRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-07-13T15:57:19Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"startTime":"2026-07-13T15:57:19Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:57:20.346Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"2b10b28d-fe07-4f98-a503-635c43f6abc7","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspended","new JobDeploymentStatus":""}
{"level":"info","ts":"2026-07-13T15:57:20.357Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"690a8083-121a-493e-aaf2-90883679e40d","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:20.357Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"690a8083-121a-493e-aaf2-90883679e40d"}
{"level":"info","ts":"2026-07-13T15:57:20.357Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"690a8083-121a-493e-aaf2-90883679e40d","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-13T15:57:20.357Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"690a8083-121a-493e-aaf2-90883679e40d","oldRayJobStatus":{"rayJobInfo":{},"startTime":"2026-07-13T15:57:19Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:57:20.357Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"690a8083-121a-493e-aaf2-90883679e40d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T15:57:20.367Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"52d72709-5967-4465-b87f-512e368b8ea5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:20.367Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"52d72709-5967-4465-b87f-512e368b8ea5","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.367Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"52d72709-5967-4465-b87f-512e368b8ea5","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:20.367Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"52d72709-5967-4465-b87f-512e368b8ea5","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:20.371Z","logger":"raycluster-resource","msg":"default","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.371Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-6m4r2","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:57:20.381Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"52d72709-5967-4465-b87f-512e368b8ea5","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:20.381Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"52d72709-5967-4465-b87f-512e368b8ea5"}
{"level":"info","ts":"2026-07-13T15:57:20.381Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"52d72709-5967-4465-b87f-512e368b8ea5","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:57:20.381Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"06137515-6e2e-492b-b21b-c4fdb25dfed9","namespacedName":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:20.381Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"9154e1fb-3bdb-426c-bb98-a27a33ca29a2","rayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.381Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"06137515-6e2e-492b-b21b-c4fdb25dfed9","mode":"IntegratedOAuth","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.381Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"06137515-6e2e-492b-b21b-c4fdb25dfed9","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.381Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"9154e1fb-3bdb-426c-bb98-a27a33ca29a2","rayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.381Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"06137515-6e2e-492b-b21b-c4fdb25dfed9","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.381Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ee3b5024-b94a-472b-9c7d-60f320976f1e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:20.381Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ee3b5024-b94a-472b-9c7d-60f320976f1e","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.381Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ee3b5024-b94a-472b-9c7d-60f320976f1e","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:20.381Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"7ca0e94d-ee08-431f-abd1-9df50131fb1d","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.381Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ee3b5024-b94a-472b-9c7d-60f320976f1e"}
{"level":"info","ts":"2026-07-13T15:57:20.381Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ee3b5024-b94a-472b-9c7d-60f320976f1e","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:57:20.381Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"34d4edb2-1210-4348-8487-258b59e3af9a"}
{"level":"info","ts":"2026-07-13T15:57:20.381Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"34d4edb2-1210-4348-8487-258b59e3af9a"}
{"level":"info","ts":"2026-07-13T15:57:20.386Z","logger":"raycluster-resource","msg":"default","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.386Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-6m4r2","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:57:20.388Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"34d4edb2-1210-4348-8487-258b59e3af9a","name":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:20.388Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"34d4edb2-1210-4348-8487-258b59e3af9a","error":"head certificate secret ray-head-secret-counter-6m4r2 not found"}
{"level":"error","ts":"2026-07-13T15:57:20.388Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"34d4edb2-1210-4348-8487-258b59e3af9a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:20.392Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"9154e1fb-3bdb-426c-bb98-a27a33ca29a2","rayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.421Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"34d4edb2-1210-4348-8487-258b59e3af9a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found"}
{"level":"info","ts":"2026-07-13T15:57:20.421Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"9154e1fb-3bdb-426c-bb98-a27a33ca29a2","rayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.421Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"34d4edb2-1210-4348-8487-258b59e3af9a","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:20.421Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"34d4edb2-1210-4348-8487-258b59e3af9a","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:20.421Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"34d4edb2-1210-4348-8487-258b59e3af9a","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:20.421Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"34d4edb2-1210-4348-8487-258b59e3af9a","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:57:20.421Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"34d4edb2-1210-4348-8487-258b59e3af9a","name":"counter-6m4r2","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:57:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:57:20.421Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"7ca0e94d-ee08-431f-abd1-9df50131fb1d","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.421Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"af8086f2-11db-44a8-83d5-c2a8a746e8df","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.422Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"b2e2f2f8-1bb0-4e31-ae51-5aaa1cccd791","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:20.422Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"b2e2f2f8-1bb0-4e31-ae51-5aaa1cccd791","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.422Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"b2e2f2f8-1bb0-4e31-ae51-5aaa1cccd791","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:20.422Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"b2e2f2f8-1bb0-4e31-ae51-5aaa1cccd791"}
{"level":"info","ts":"2026-07-13T15:57:20.422Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"b2e2f2f8-1bb0-4e31-ae51-5aaa1cccd791","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:57:20.426Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"06137515-6e2e-492b-b21b-c4fdb25dfed9","name":"counter-6m4r2-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:57:20.427Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"34d4edb2-1210-4348-8487-258b59e3af9a","name":"counter-6m4r2","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-6m4r2\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-88gnn","name":"counter-6m4r2"}}
{"level":"info","ts":"2026-07-13T15:57:20.427Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"34d4edb2-1210-4348-8487-258b59e3af9a"}
{"level":"error","ts":"2026-07-13T15:57:20.427Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"34d4edb2-1210-4348-8487-258b59e3af9a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:20.428Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78fe8c75-4049-424b-a1ef-95d192f92037"}
{"level":"info","ts":"2026-07-13T15:57:20.428Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78fe8c75-4049-424b-a1ef-95d192f92037"}
{"level":"info","ts":"2026-07-13T15:57:20.428Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78fe8c75-4049-424b-a1ef-95d192f92037","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:20.428Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78fe8c75-4049-424b-a1ef-95d192f92037","error":"head certificate secret ray-head-secret-counter-6m4r2 not found"}
{"level":"error","ts":"2026-07-13T15:57:20.428Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78fe8c75-4049-424b-a1ef-95d192f92037","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:20.428Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78fe8c75-4049-424b-a1ef-95d192f92037","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found"}
{"level":"info","ts":"2026-07-13T15:57:20.428Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78fe8c75-4049-424b-a1ef-95d192f92037","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:20.428Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78fe8c75-4049-424b-a1ef-95d192f92037","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:20.428Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78fe8c75-4049-424b-a1ef-95d192f92037","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:20.428Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78fe8c75-4049-424b-a1ef-95d192f92037","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:57:20.428Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78fe8c75-4049-424b-a1ef-95d192f92037","name":"counter-6m4r2","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:57:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:57:20.431Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"af8086f2-11db-44a8-83d5-c2a8a746e8df","name":"counter-6m4r2-head"}
{"level":"info","ts":"2026-07-13T15:57:20.432Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"9154e1fb-3bdb-426c-bb98-a27a33ca29a2","rayCluster":"counter-6m4r2","secret":"ca-secret"}
{"level":"info","ts":"2026-07-13T15:57:20.432Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"9154e1fb-3bdb-426c-bb98-a27a33ca29a2","rayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.435Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"06137515-6e2e-492b-b21b-c4fdb25dfed9","name":"kuberay-gateway-access","namespace":"test-ns-88gnn","operation":"created"}
{"level":"info","ts":"2026-07-13T15:57:20.438Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78fe8c75-4049-424b-a1ef-95d192f92037"}
{"level":"error","ts":"2026-07-13T15:57:20.438Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78fe8c75-4049-424b-a1ef-95d192f92037","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:20.438Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78a42b79-601f-4593-a5b9-9d080faf1f94"}
{"level":"info","ts":"2026-07-13T15:57:20.438Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78a42b79-601f-4593-a5b9-9d080faf1f94"}
{"level":"info","ts":"2026-07-13T15:57:20.438Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78a42b79-601f-4593-a5b9-9d080faf1f94","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:20.438Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78a42b79-601f-4593-a5b9-9d080faf1f94","error":"head certificate secret ray-head-secret-counter-6m4r2 not found"}
{"level":"error","ts":"2026-07-13T15:57:20.438Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78a42b79-601f-4593-a5b9-9d080faf1f94","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:20.438Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78a42b79-601f-4593-a5b9-9d080faf1f94","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found"}
{"level":"info","ts":"2026-07-13T15:57:20.438Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78a42b79-601f-4593-a5b9-9d080faf1f94","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:20.438Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78a42b79-601f-4593-a5b9-9d080faf1f94","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:20.438Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78a42b79-601f-4593-a5b9-9d080faf1f94","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:20.438Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78a42b79-601f-4593-a5b9-9d080faf1f94","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:57:20.438Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78a42b79-601f-4593-a5b9-9d080faf1f94","name":"counter-6m4r2","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:57:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:57:20.439Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"7b5c9108-9437-4913-abbe-d995a7123171","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:20.439Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"7b5c9108-9437-4913-abbe-d995a7123171","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.439Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"7b5c9108-9437-4913-abbe-d995a7123171","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:20.439Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"7b5c9108-9437-4913-abbe-d995a7123171"}
{"level":"info","ts":"2026-07-13T15:57:20.439Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"7b5c9108-9437-4913-abbe-d995a7123171","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:57:20.442Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"9154e1fb-3bdb-426c-bb98-a27a33ca29a2","rayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.521Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"af8086f2-11db-44a8-83d5-c2a8a746e8df","name":"counter-6m4r2-workers"}
{"level":"info","ts":"2026-07-13T15:57:20.521Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"af8086f2-11db-44a8-83d5-c2a8a746e8df","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.521Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"06137515-6e2e-492b-b21b-c4fdb25dfed9","name":"test-ns-88gnn-counter-6m4r2","operation":"created"}
{"level":"info","ts":"2026-07-13T15:57:20.521Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"9154e1fb-3bdb-426c-bb98-a27a33ca29a2","rayCluster":"counter-6m4r2","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:57:20.521Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78a42b79-601f-4593-a5b9-9d080faf1f94","name":"counter-6m4r2","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-6m4r2\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-88gnn","name":"counter-6m4r2"}}
{"level":"info","ts":"2026-07-13T15:57:20.521Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78a42b79-601f-4593-a5b9-9d080faf1f94"}
{"level":"error","ts":"2026-07-13T15:57:20.521Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"78a42b79-601f-4593-a5b9-9d080faf1f94","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:20.521Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"f0546112-8c7d-43dd-aa47-adc0630af82f","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.522Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"597456e5-cb2b-466c-93c3-08360dfa3d58"}
{"level":"info","ts":"2026-07-13T15:57:20.522Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"597456e5-cb2b-466c-93c3-08360dfa3d58"}
{"level":"info","ts":"2026-07-13T15:57:20.522Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"597456e5-cb2b-466c-93c3-08360dfa3d58","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:20.522Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"597456e5-cb2b-466c-93c3-08360dfa3d58","error":"head certificate secret ray-head-secret-counter-6m4r2 not found"}
{"level":"error","ts":"2026-07-13T15:57:20.522Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"597456e5-cb2b-466c-93c3-08360dfa3d58","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:20.522Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"597456e5-cb2b-466c-93c3-08360dfa3d58","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found"}
{"level":"info","ts":"2026-07-13T15:57:20.522Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"597456e5-cb2b-466c-93c3-08360dfa3d58","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:20.522Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"597456e5-cb2b-466c-93c3-08360dfa3d58","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:20.522Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"597456e5-cb2b-466c-93c3-08360dfa3d58","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:20.522Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"597456e5-cb2b-466c-93c3-08360dfa3d58"}
{"level":"error","ts":"2026-07-13T15:57:20.522Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"597456e5-cb2b-466c-93c3-08360dfa3d58","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:20.529Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"06137515-6e2e-492b-b21b-c4fdb25dfed9","name":"kube-rbac-proxy-config-counter-6m4r2","operation":"created"}
{"level":"info","ts":"2026-07-13T15:57:20.532Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"9154e1fb-3bdb-426c-bb98-a27a33ca29a2","rayCluster":"counter-6m4r2","podIPs":[]}
{"level":"error","ts":"2026-07-13T15:57:20.532Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"9154e1fb-3bdb-426c-bb98-a27a33ca29a2","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-6m4r2\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:20.532Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"9154e1fb-3bdb-426c-bb98-a27a33ca29a2"}
{"level":"error","ts":"2026-07-13T15:57:20.532Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"9154e1fb-3bdb-426c-bb98-a27a33ca29a2","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-6m4r2\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:20.533Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"1414463d-b051-4ae8-ac05-d463dbc2c5f6","rayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.533Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"1414463d-b051-4ae8-ac05-d463dbc2c5f6"}
{"level":"error","ts":"2026-07-13T15:57:20.535Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"06137515-6e2e-492b-b21b-c4fdb25dfed9","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-6m4r2\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:20.535Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"06137515-6e2e-492b-b21b-c4fdb25dfed9"}
{"level":"error","ts":"2026-07-13T15:57:20.535Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"06137515-6e2e-492b-b21b-c4fdb25dfed9","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-6m4r2\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:20.535Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"4a6e616a-bf84-43b5-a89a-3156e4aee3ef","namespacedName":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:20.535Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"4a6e616a-bf84-43b5-a89a-3156e4aee3ef","mode":"IntegratedOAuth","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.535Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"4a6e616a-bf84-43b5-a89a-3156e4aee3ef","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.535Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"f0546112-8c7d-43dd-aa47-adc0630af82f","name":"counter-6m4r2-head"}
{"level":"info","ts":"2026-07-13T15:57:20.539Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"38beb351-3e94-4e27-8500-525e6e10555d","rayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.539Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"38beb351-3e94-4e27-8500-525e6e10555d"}
{"level":"info","ts":"2026-07-13T15:57:20.542Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"e804709b-57d0-4935-b642-32b83d4d0367"}
{"level":"info","ts":"2026-07-13T15:57:20.542Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"e804709b-57d0-4935-b642-32b83d4d0367"}
{"level":"info","ts":"2026-07-13T15:57:20.542Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"e804709b-57d0-4935-b642-32b83d4d0367","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:20.542Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"e804709b-57d0-4935-b642-32b83d4d0367","error":"head certificate secret ray-head-secret-counter-6m4r2 not found"}
{"level":"error","ts":"2026-07-13T15:57:20.542Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"e804709b-57d0-4935-b642-32b83d4d0367","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:20.542Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"e804709b-57d0-4935-b642-32b83d4d0367","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found"}
{"level":"info","ts":"2026-07-13T15:57:20.542Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"e804709b-57d0-4935-b642-32b83d4d0367","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:20.542Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"e804709b-57d0-4935-b642-32b83d4d0367","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:20.542Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"e804709b-57d0-4935-b642-32b83d4d0367","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:20.542Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"e804709b-57d0-4935-b642-32b83d4d0367"}
{"level":"error","ts":"2026-07-13T15:57:20.542Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"e804709b-57d0-4935-b642-32b83d4d0367","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:20.622Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"f0546112-8c7d-43dd-aa47-adc0630af82f","name":"counter-6m4r2-workers"}
{"level":"info","ts":"2026-07-13T15:57:20.622Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"f0546112-8c7d-43dd-aa47-adc0630af82f","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.622Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"4a6e616a-bf84-43b5-a89a-3156e4aee3ef","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.622Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"1d6c9376-fae7-44e1-adb3-312653db7f0a","namespacedName":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:20.622Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"1d6c9376-fae7-44e1-adb3-312653db7f0a","mode":"IntegratedOAuth","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.622Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"1d6c9376-fae7-44e1-adb3-312653db7f0a","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.622Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"cbb32245-469a-404f-be9b-b3fcc61f6b35","rayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.622Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"cbb32245-469a-404f-be9b-b3fcc61f6b35"}
{"level":"info","ts":"2026-07-13T15:57:20.622Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"03a3d285-bf7d-43f4-931e-4de95580ead6","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.622Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"65c1bbf4-9984-4b83-9dbb-4400e87fc097","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:20.622Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"65c1bbf4-9984-4b83-9dbb-4400e87fc097","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.622Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"65c1bbf4-9984-4b83-9dbb-4400e87fc097","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:20.622Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1d0480af-22d9-4b07-ad4d-5400ceaeb2e1"}
{"level":"info","ts":"2026-07-13T15:57:20.623Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1d0480af-22d9-4b07-ad4d-5400ceaeb2e1"}
{"level":"info","ts":"2026-07-13T15:57:20.623Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1d0480af-22d9-4b07-ad4d-5400ceaeb2e1","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:20.623Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"65c1bbf4-9984-4b83-9dbb-4400e87fc097"}
{"level":"info","ts":"2026-07-13T15:57:20.623Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"65c1bbf4-9984-4b83-9dbb-4400e87fc097","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:57:20.623Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1d0480af-22d9-4b07-ad4d-5400ceaeb2e1","error":"head certificate secret ray-head-secret-counter-6m4r2 not found"}
{"level":"error","ts":"2026-07-13T15:57:20.623Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1d0480af-22d9-4b07-ad4d-5400ceaeb2e1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:20.623Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1d0480af-22d9-4b07-ad4d-5400ceaeb2e1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found"}
{"level":"info","ts":"2026-07-13T15:57:20.623Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1d0480af-22d9-4b07-ad4d-5400ceaeb2e1","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:20.623Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1d0480af-22d9-4b07-ad4d-5400ceaeb2e1","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:20.623Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1d0480af-22d9-4b07-ad4d-5400ceaeb2e1","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:20.623Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1d0480af-22d9-4b07-ad4d-5400ceaeb2e1"}
{"level":"error","ts":"2026-07-13T15:57:20.623Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1d0480af-22d9-4b07-ad4d-5400ceaeb2e1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:20.631Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"1d6c9376-fae7-44e1-adb3-312653db7f0a","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.634Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"03a3d285-bf7d-43f4-931e-4de95580ead6","name":"counter-6m4r2-head"}
{"level":"info","ts":"2026-07-13T15:57:20.647Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"03a3d285-bf7d-43f4-931e-4de95580ead6","name":"counter-6m4r2-workers"}
{"level":"info","ts":"2026-07-13T15:57:20.647Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"03a3d285-bf7d-43f4-931e-4de95580ead6","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:20.784Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a6c09bce-325b-4e67-a466-5e54dac5357c"}
{"level":"info","ts":"2026-07-13T15:57:20.784Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a6c09bce-325b-4e67-a466-5e54dac5357c"}
{"level":"info","ts":"2026-07-13T15:57:20.784Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a6c09bce-325b-4e67-a466-5e54dac5357c","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:20.784Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a6c09bce-325b-4e67-a466-5e54dac5357c","error":"head certificate secret ray-head-secret-counter-6m4r2 not found"}
{"level":"error","ts":"2026-07-13T15:57:20.784Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a6c09bce-325b-4e67-a466-5e54dac5357c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:20.784Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a6c09bce-325b-4e67-a466-5e54dac5357c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found"}
{"level":"info","ts":"2026-07-13T15:57:20.784Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a6c09bce-325b-4e67-a466-5e54dac5357c","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:20.784Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a6c09bce-325b-4e67-a466-5e54dac5357c","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:20.784Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a6c09bce-325b-4e67-a466-5e54dac5357c","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:20.784Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a6c09bce-325b-4e67-a466-5e54dac5357c"}
{"level":"error","ts":"2026-07-13T15:57:20.784Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a6c09bce-325b-4e67-a466-5e54dac5357c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:21.105Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9d2d2af-7fef-46cd-9dea-3d72409bd52f"}
{"level":"info","ts":"2026-07-13T15:57:21.105Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9d2d2af-7fef-46cd-9dea-3d72409bd52f"}
{"level":"info","ts":"2026-07-13T15:57:21.105Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9d2d2af-7fef-46cd-9dea-3d72409bd52f","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:21.105Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9d2d2af-7fef-46cd-9dea-3d72409bd52f","error":"head certificate secret ray-head-secret-counter-6m4r2 not found"}
{"level":"error","ts":"2026-07-13T15:57:21.105Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9d2d2af-7fef-46cd-9dea-3d72409bd52f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:21.105Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9d2d2af-7fef-46cd-9dea-3d72409bd52f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found"}
{"level":"info","ts":"2026-07-13T15:57:21.105Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9d2d2af-7fef-46cd-9dea-3d72409bd52f","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:21.105Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9d2d2af-7fef-46cd-9dea-3d72409bd52f","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:21.105Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9d2d2af-7fef-46cd-9dea-3d72409bd52f","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:21.105Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9d2d2af-7fef-46cd-9dea-3d72409bd52f"}
{"level":"error","ts":"2026-07-13T15:57:21.105Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9d2d2af-7fef-46cd-9dea-3d72409bd52f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:21.746Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b9e1594a-dc95-4065-b412-c40bf5d527a9"}
{"level":"info","ts":"2026-07-13T15:57:21.746Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b9e1594a-dc95-4065-b412-c40bf5d527a9"}
{"level":"info","ts":"2026-07-13T15:57:21.746Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b9e1594a-dc95-4065-b412-c40bf5d527a9","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:21.746Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b9e1594a-dc95-4065-b412-c40bf5d527a9","error":"head certificate secret ray-head-secret-counter-6m4r2 not found"}
{"level":"error","ts":"2026-07-13T15:57:21.746Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b9e1594a-dc95-4065-b412-c40bf5d527a9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:21.746Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b9e1594a-dc95-4065-b412-c40bf5d527a9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found"}
{"level":"info","ts":"2026-07-13T15:57:21.746Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b9e1594a-dc95-4065-b412-c40bf5d527a9","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:21.746Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b9e1594a-dc95-4065-b412-c40bf5d527a9","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:21.746Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b9e1594a-dc95-4065-b412-c40bf5d527a9","filter labels":["test-ns-88gnn",{"ray.io/cluster":"counter-6m4r2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:57:21.746Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b9e1594a-dc95-4065-b412-c40bf5d527a9"}
{"level":"error","ts":"2026-07-13T15:57:21.746Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b9e1594a-dc95-4065-b412-c40bf5d527a9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6m4r2 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:22.622Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"558664ae-1ce7-4963-9a77-7c6d6dd9dd2e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:22.623Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"558664ae-1ce7-4963-9a77-7c6d6dd9dd2e","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:22.623Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"558664ae-1ce7-4963-9a77-7c6d6dd9dd2e","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:22.623Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"558664ae-1ce7-4963-9a77-7c6d6dd9dd2e"}
{"level":"info","ts":"2026-07-13T15:57:22.623Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"558664ae-1ce7-4963-9a77-7c6d6dd9dd2e","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:57:23.027Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c"}
{"level":"info","ts":"2026-07-13T15:57:23.027Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c"}
{"level":"info","ts":"2026-07-13T15:57:23.027Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:23.027Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c"}
{"level":"info","ts":"2026-07-13T15:57:23.027Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:57:23.027Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c"}
{"level":"info","ts":"2026-07-13T15:57:23.027Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","mode":"IntegratedOAuth","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:23.027Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","cluster":"counter-6m4r2","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:57:23.027Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","cluster":"counter-6m4r2","authType":"IntegratedOAuth","serviceAccount":"counter-6m4r2-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T15:57:23.027Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-6m4r2","ray.io/group":"headgroup","ray.io/identifier":"counter-6m4r2-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T15:57:23.027Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T15:57:23.028Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:57:23.028Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:57:23.028Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","enabled":true}
{"level":"info","ts":"2026-07-13T15:57:23.041Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","name":"counter-6m4r2-head-767d2"}
{"level":"info","ts":"2026-07-13T15:57:23.041Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:57:23.041Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:23.041Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T15:57:23.041Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:23.041Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T15:57:23.041Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c"}
{"level":"info","ts":"2026-07-13T15:57:23.041Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","nodeType":"worker","rayStartParams":{"address":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T15:57:23.041Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","rayStartCmd":"ray start  --address=counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:57:23.041Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:57:23.041Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","enabled":true}
{"level":"info","ts":"2026-07-13T15:57:23.055Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","name":"counter-6m4r2-small-group-worker-nlscq"}
{"level":"info","ts":"2026-07-13T15:57:23.055Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","podName":"counter-6m4r2-head-767d2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:23.055Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-6m4r2-head-svc"},"newHead":{"podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"}}
{"level":"info","ts":"2026-07-13T15:57:23.055Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"177e6200-74c2-4ed2-a11d-f3d4afae184c","name":"counter-6m4r2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:57:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:57:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:57:23.068Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b439319e-727b-434c-9847-3a126f381661"}
{"level":"info","ts":"2026-07-13T15:57:23.068Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b439319e-727b-434c-9847-3a126f381661"}
{"level":"info","ts":"2026-07-13T15:57:23.068Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b439319e-727b-434c-9847-3a126f381661","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:23.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b439319e-727b-434c-9847-3a126f381661","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:57:23.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b439319e-727b-434c-9847-3a126f381661","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:23.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b439319e-727b-434c-9847-3a126f381661","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:57:23.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b439319e-727b-434c-9847-3a126f381661","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:23.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b439319e-727b-434c-9847-3a126f381661","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:23.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b439319e-727b-434c-9847-3a126f381661","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:57:23.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b439319e-727b-434c-9847-3a126f381661","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:23.068Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b439319e-727b-434c-9847-3a126f381661","podName":"counter-6m4r2-head-767d2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:23.068Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b439319e-727b-434c-9847-3a126f381661","podName":"counter-6m4r2-head-767d2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:23.068Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b439319e-727b-434c-9847-3a126f381661","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-6m4r2-head-svc"},"newHead":{"podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"}}
{"level":"info","ts":"2026-07-13T15:57:23.068Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b439319e-727b-434c-9847-3a126f381661","name":"counter-6m4r2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:57:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:57:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:57:23.068Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"dff9f679-5d4d-46f6-b427-1d7c8b74749a","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:23.068Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"9f9873b8-0aa5-4ac0-9787-9319b9661634","rayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:23.068Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"521a250b-124c-46d9-a5be-dda39b639814","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:23.068Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"521a250b-124c-46d9-a5be-dda39b639814","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:23.069Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"521a250b-124c-46d9-a5be-dda39b639814","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:23.069Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"9f9873b8-0aa5-4ac0-9787-9319b9661634","rayCluster":"counter-6m4r2","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:57:23.069Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"521a250b-124c-46d9-a5be-dda39b639814"}
{"level":"info","ts":"2026-07-13T15:57:23.069Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"521a250b-124c-46d9-a5be-dda39b639814","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:57:23.075Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b439319e-727b-434c-9847-3a126f381661","name":"counter-6m4r2","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-6m4r2\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-88gnn","name":"counter-6m4r2"}}
{"level":"info","ts":"2026-07-13T15:57:23.076Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b439319e-727b-434c-9847-3a126f381661"}
{"level":"error","ts":"2026-07-13T15:57:23.076Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b439319e-727b-434c-9847-3a126f381661","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-6m4r2\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:23.076Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bdc59cc7-6472-47ca-bf5d-973282ee4f78"}
{"level":"info","ts":"2026-07-13T15:57:23.076Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bdc59cc7-6472-47ca-bf5d-973282ee4f78"}
{"level":"info","ts":"2026-07-13T15:57:23.076Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bdc59cc7-6472-47ca-bf5d-973282ee4f78","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:23.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bdc59cc7-6472-47ca-bf5d-973282ee4f78","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:57:23.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bdc59cc7-6472-47ca-bf5d-973282ee4f78","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:23.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bdc59cc7-6472-47ca-bf5d-973282ee4f78","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:57:23.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bdc59cc7-6472-47ca-bf5d-973282ee4f78","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:23.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bdc59cc7-6472-47ca-bf5d-973282ee4f78","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:23.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bdc59cc7-6472-47ca-bf5d-973282ee4f78","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:57:23.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bdc59cc7-6472-47ca-bf5d-973282ee4f78","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:23.121Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bdc59cc7-6472-47ca-bf5d-973282ee4f78","podName":"counter-6m4r2-head-767d2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:23.121Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bdc59cc7-6472-47ca-bf5d-973282ee4f78","podName":"counter-6m4r2-head-767d2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:23.121Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bdc59cc7-6472-47ca-bf5d-973282ee4f78","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:57:23.121Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bdc59cc7-6472-47ca-bf5d-973282ee4f78","seconds":300}
{"level":"info","ts":"2026-07-13T15:57:23.121Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c3c2831d-5766-49ea-bdb5-efeeba80080e"}
{"level":"info","ts":"2026-07-13T15:57:23.121Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c3c2831d-5766-49ea-bdb5-efeeba80080e"}
{"level":"info","ts":"2026-07-13T15:57:23.121Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c3c2831d-5766-49ea-bdb5-efeeba80080e","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:23.121Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c3c2831d-5766-49ea-bdb5-efeeba80080e","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:57:23.121Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c3c2831d-5766-49ea-bdb5-efeeba80080e","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:23.121Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c3c2831d-5766-49ea-bdb5-efeeba80080e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:57:23.121Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c3c2831d-5766-49ea-bdb5-efeeba80080e","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:23.121Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c3c2831d-5766-49ea-bdb5-efeeba80080e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:23.121Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c3c2831d-5766-49ea-bdb5-efeeba80080e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:57:23.121Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c3c2831d-5766-49ea-bdb5-efeeba80080e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:23.121Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c3c2831d-5766-49ea-bdb5-efeeba80080e","podName":"counter-6m4r2-head-767d2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:23.121Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c3c2831d-5766-49ea-bdb5-efeeba80080e","podName":"counter-6m4r2-head-767d2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:23.121Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c3c2831d-5766-49ea-bdb5-efeeba80080e","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:57:23.121Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c3c2831d-5766-49ea-bdb5-efeeba80080e","name":"counter-6m4r2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:57:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:57:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:57:23.125Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"dff9f679-5d4d-46f6-b427-1d7c8b74749a","name":"counter-6m4r2-head"}
{"level":"info","ts":"2026-07-13T15:57:23.133Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6ad3ee3a-a2f6-4d47-808e-9b74c1668dc5"}
{"level":"info","ts":"2026-07-13T15:57:23.133Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6ad3ee3a-a2f6-4d47-808e-9b74c1668dc5"}
{"level":"info","ts":"2026-07-13T15:57:23.133Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6ad3ee3a-a2f6-4d47-808e-9b74c1668dc5","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:23.133Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"a9dee165-9c8d-475c-aba6-81270e0aab73","rayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:23.133Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6ad3ee3a-a2f6-4d47-808e-9b74c1668dc5","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:57:23.133Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6ad3ee3a-a2f6-4d47-808e-9b74c1668dc5","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:23.133Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6ad3ee3a-a2f6-4d47-808e-9b74c1668dc5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:57:23.133Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6ad3ee3a-a2f6-4d47-808e-9b74c1668dc5","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:23.133Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6ad3ee3a-a2f6-4d47-808e-9b74c1668dc5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:23.133Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6ad3ee3a-a2f6-4d47-808e-9b74c1668dc5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:57:23.133Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6ad3ee3a-a2f6-4d47-808e-9b74c1668dc5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:23.133Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"95ab1af8-21ee-47f3-b118-a5b0c7ea0999","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:23.133Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"95ab1af8-21ee-47f3-b118-a5b0c7ea0999","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:23.133Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"a9dee165-9c8d-475c-aba6-81270e0aab73","rayCluster":"counter-6m4r2","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:57:23.133Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"95ab1af8-21ee-47f3-b118-a5b0c7ea0999","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:23.133Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6ad3ee3a-a2f6-4d47-808e-9b74c1668dc5","podName":"counter-6m4r2-head-767d2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:23.133Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6ad3ee3a-a2f6-4d47-808e-9b74c1668dc5","podName":"counter-6m4r2-head-767d2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:23.133Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"95ab1af8-21ee-47f3-b118-a5b0c7ea0999"}
{"level":"info","ts":"2026-07-13T15:57:23.133Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"95ab1af8-21ee-47f3-b118-a5b0c7ea0999","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:57:23.133Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6ad3ee3a-a2f6-4d47-808e-9b74c1668dc5","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:57:23.133Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6ad3ee3a-a2f6-4d47-808e-9b74c1668dc5","name":"counter-6m4r2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:57:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:57:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:57:23.136Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"dff9f679-5d4d-46f6-b427-1d7c8b74749a","name":"counter-6m4r2-workers"}
{"level":"info","ts":"2026-07-13T15:57:23.136Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"dff9f679-5d4d-46f6-b427-1d7c8b74749a","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:23.136Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"1b219e8a-4350-446d-9c4a-8eedbcefa780","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:23.138Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6ad3ee3a-a2f6-4d47-808e-9b74c1668dc5","name":"counter-6m4r2","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-6m4r2\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-88gnn","name":"counter-6m4r2"}}
{"level":"info","ts":"2026-07-13T15:57:23.138Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6ad3ee3a-a2f6-4d47-808e-9b74c1668dc5"}
{"level":"error","ts":"2026-07-13T15:57:23.138Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6ad3ee3a-a2f6-4d47-808e-9b74c1668dc5","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-6m4r2\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:57:23.143Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"82837d04-0cfd-432a-8a52-4e66b076f2a2"}
{"level":"info","ts":"2026-07-13T15:57:23.143Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"82837d04-0cfd-432a-8a52-4e66b076f2a2"}
{"level":"info","ts":"2026-07-13T15:57:23.143Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"82837d04-0cfd-432a-8a52-4e66b076f2a2","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:23.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"82837d04-0cfd-432a-8a52-4e66b076f2a2","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:57:23.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"82837d04-0cfd-432a-8a52-4e66b076f2a2","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:23.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"82837d04-0cfd-432a-8a52-4e66b076f2a2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:57:23.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"82837d04-0cfd-432a-8a52-4e66b076f2a2","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:23.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"82837d04-0cfd-432a-8a52-4e66b076f2a2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:23.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"82837d04-0cfd-432a-8a52-4e66b076f2a2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:57:23.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"82837d04-0cfd-432a-8a52-4e66b076f2a2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:23.143Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"82837d04-0cfd-432a-8a52-4e66b076f2a2","podName":"counter-6m4r2-head-767d2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:23.143Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"82837d04-0cfd-432a-8a52-4e66b076f2a2","podName":"counter-6m4r2-head-767d2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:23.143Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"82837d04-0cfd-432a-8a52-4e66b076f2a2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:57:23.143Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"82837d04-0cfd-432a-8a52-4e66b076f2a2","seconds":300}
{"level":"info","ts":"2026-07-13T15:57:23.146Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"1b219e8a-4350-446d-9c4a-8eedbcefa780","name":"counter-6m4r2-head"}
{"level":"info","ts":"2026-07-13T15:57:23.154Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"1b219e8a-4350-446d-9c4a-8eedbcefa780","name":"counter-6m4r2-workers"}
{"level":"info","ts":"2026-07-13T15:57:23.154Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"1b219e8a-4350-446d-9c4a-8eedbcefa780","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:23.492Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"acb575a2-0fd7-43b8-bf41-b6ec3a5b0e10"}
{"level":"info","ts":"2026-07-13T15:57:23.492Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"acb575a2-0fd7-43b8-bf41-b6ec3a5b0e10"}
{"level":"info","ts":"2026-07-13T15:57:23.492Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"acb575a2-0fd7-43b8-bf41-b6ec3a5b0e10","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:23.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"acb575a2-0fd7-43b8-bf41-b6ec3a5b0e10","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:57:23.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"acb575a2-0fd7-43b8-bf41-b6ec3a5b0e10","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:23.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"acb575a2-0fd7-43b8-bf41-b6ec3a5b0e10","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:57:23.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"acb575a2-0fd7-43b8-bf41-b6ec3a5b0e10","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:23.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"acb575a2-0fd7-43b8-bf41-b6ec3a5b0e10","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:23.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"acb575a2-0fd7-43b8-bf41-b6ec3a5b0e10","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:57:23.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"acb575a2-0fd7-43b8-bf41-b6ec3a5b0e10","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:23.492Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"acb575a2-0fd7-43b8-bf41-b6ec3a5b0e10","podName":"counter-6m4r2-head-767d2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:23.492Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"acb575a2-0fd7-43b8-bf41-b6ec3a5b0e10","podName":"counter-6m4r2-head-767d2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:23.492Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"acb575a2-0fd7-43b8-bf41-b6ec3a5b0e10","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:57:23.492Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"acb575a2-0fd7-43b8-bf41-b6ec3a5b0e10","seconds":300}
{"level":"info","ts":"2026-07-13T15:57:23.502Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c7e97749-085e-4278-979a-6bc068445b44"}
{"level":"info","ts":"2026-07-13T15:57:23.502Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c7e97749-085e-4278-979a-6bc068445b44"}
{"level":"info","ts":"2026-07-13T15:57:23.502Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c7e97749-085e-4278-979a-6bc068445b44","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:23.502Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c7e97749-085e-4278-979a-6bc068445b44","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:57:23.502Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c7e97749-085e-4278-979a-6bc068445b44","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:23.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c7e97749-085e-4278-979a-6bc068445b44","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:57:23.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c7e97749-085e-4278-979a-6bc068445b44","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:23.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c7e97749-085e-4278-979a-6bc068445b44","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:23.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c7e97749-085e-4278-979a-6bc068445b44","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:57:23.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c7e97749-085e-4278-979a-6bc068445b44","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:23.503Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c7e97749-085e-4278-979a-6bc068445b44","podName":"counter-6m4r2-head-767d2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:23.503Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c7e97749-085e-4278-979a-6bc068445b44","podName":"counter-6m4r2-head-767d2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:23.503Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c7e97749-085e-4278-979a-6bc068445b44","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:57:23.503Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"c7e97749-085e-4278-979a-6bc068445b44","seconds":300}
{"level":"info","ts":"2026-07-13T15:57:23.595Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bde476e0-36a2-4bae-87a8-0e48b971a27e"}
{"level":"info","ts":"2026-07-13T15:57:23.595Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bde476e0-36a2-4bae-87a8-0e48b971a27e"}
{"level":"info","ts":"2026-07-13T15:57:23.595Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bde476e0-36a2-4bae-87a8-0e48b971a27e","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:23.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bde476e0-36a2-4bae-87a8-0e48b971a27e","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:57:23.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bde476e0-36a2-4bae-87a8-0e48b971a27e","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:23.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bde476e0-36a2-4bae-87a8-0e48b971a27e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:57:23.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bde476e0-36a2-4bae-87a8-0e48b971a27e","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:23.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bde476e0-36a2-4bae-87a8-0e48b971a27e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:23.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bde476e0-36a2-4bae-87a8-0e48b971a27e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:57:23.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bde476e0-36a2-4bae-87a8-0e48b971a27e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:23.595Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bde476e0-36a2-4bae-87a8-0e48b971a27e","podName":"counter-6m4r2-head-767d2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:23.595Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bde476e0-36a2-4bae-87a8-0e48b971a27e","podName":"counter-6m4r2-head-767d2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:23.595Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bde476e0-36a2-4bae-87a8-0e48b971a27e","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"newHead":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"}}
{"level":"info","ts":"2026-07-13T15:57:23.595Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bde476e0-36a2-4bae-87a8-0e48b971a27e","name":"counter-6m4r2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:57:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:57:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:57:23.606Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"6397e1ec-ba3f-4b08-8643-a8d8ad46ea14","rayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:23.606Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"b509c178-4b6e-4e97-85ad-dee47d6e256e","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:23.606Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"6397e1ec-ba3f-4b08-8643-a8d8ad46ea14","pod":"counter-6m4r2-head-767d2","ip":"10.133.0.26"}
{"level":"info","ts":"2026-07-13T15:57:23.606Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"2b410b35-12a9-4d7c-85cd-c37eeb836f9d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:23.606Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"2b410b35-12a9-4d7c-85cd-c37eeb836f9d","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:23.606Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"2b410b35-12a9-4d7c-85cd-c37eeb836f9d","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:23.606Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"2b410b35-12a9-4d7c-85cd-c37eeb836f9d"}
{"level":"info","ts":"2026-07-13T15:57:23.606Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"2b410b35-12a9-4d7c-85cd-c37eeb836f9d","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:57:23.616Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"b509c178-4b6e-4e97-85ad-dee47d6e256e","name":"counter-6m4r2-head"}
{"level":"info","ts":"2026-07-13T15:57:23.617Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"6397e1ec-ba3f-4b08-8643-a8d8ad46ea14","certificate":"ray-head-cert-counter-6m4r2","dnsNames":["counter-6m4r2-head-svc","localhost","counter-6m4r2-head-svc.test-ns-88gnn.svc","counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local"],"ipAddresses":["10.133.0.26","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:57:23.627Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"b509c178-4b6e-4e97-85ad-dee47d6e256e","name":"counter-6m4r2-workers"}
{"level":"info","ts":"2026-07-13T15:57:23.627Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"b509c178-4b6e-4e97-85ad-dee47d6e256e","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:23.628Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"6397e1ec-ba3f-4b08-8643-a8d8ad46ea14","certificate":"ray-worker-cert-counter-6m4r2","dnsNames":["counter-6m4r2-worker-svc","localhost","counter-6m4r2-small-group","counter-6m4r2-worker-svc.test-ns-88gnn.svc","counter-6m4r2-worker-svc.test-ns-88gnn.svc.cluster.local","*.counter-6m4r2-worker-svc.test-ns-88gnn.svc","*.counter-6m4r2-worker-svc.test-ns-88gnn.svc.cluster.local","*-worker-*.test-ns-88gnn.svc","*-worker-*.test-ns-88gnn.svc.cluster.local"],"ipAddresses":["10.133.0.26","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:57:23.628Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"6397e1ec-ba3f-4b08-8643-a8d8ad46ea14","rayCluster":"counter-6m4r2","podIPs":["10.133.0.26"]}
{"level":"info","ts":"2026-07-13T15:57:23.803Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1f7944cb-03e9-4ae2-9de4-2e338df7da38"}
{"level":"info","ts":"2026-07-13T15:57:23.803Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1f7944cb-03e9-4ae2-9de4-2e338df7da38"}
{"level":"info","ts":"2026-07-13T15:57:23.803Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1f7944cb-03e9-4ae2-9de4-2e338df7da38","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:23.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1f7944cb-03e9-4ae2-9de4-2e338df7da38","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:57:23.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1f7944cb-03e9-4ae2-9de4-2e338df7da38","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:23.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1f7944cb-03e9-4ae2-9de4-2e338df7da38","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:57:23.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1f7944cb-03e9-4ae2-9de4-2e338df7da38","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:23.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1f7944cb-03e9-4ae2-9de4-2e338df7da38","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:23.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1f7944cb-03e9-4ae2-9de4-2e338df7da38","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:57:23.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1f7944cb-03e9-4ae2-9de4-2e338df7da38","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:23.804Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1f7944cb-03e9-4ae2-9de4-2e338df7da38","podName":"counter-6m4r2-head-767d2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:23.804Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1f7944cb-03e9-4ae2-9de4-2e338df7da38","podName":"counter-6m4r2-head-767d2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:23.804Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1f7944cb-03e9-4ae2-9de4-2e338df7da38","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:57:23.804Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"1f7944cb-03e9-4ae2-9de4-2e338df7da38","seconds":300}
{"level":"info","ts":"2026-07-13T15:57:25.606Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"9db16090-9ae8-49c9-8276-df9abd02ae31"}
{"level":"info","ts":"2026-07-13T15:57:25.606Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"9db16090-9ae8-49c9-8276-df9abd02ae31"}
{"level":"info","ts":"2026-07-13T15:57:25.606Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"9db16090-9ae8-49c9-8276-df9abd02ae31","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:57:25.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"9db16090-9ae8-49c9-8276-df9abd02ae31","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:57:25.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"9db16090-9ae8-49c9-8276-df9abd02ae31","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:25.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"9db16090-9ae8-49c9-8276-df9abd02ae31","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:57:25.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"9db16090-9ae8-49c9-8276-df9abd02ae31","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:57:25.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"9db16090-9ae8-49c9-8276-df9abd02ae31","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:25.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"9db16090-9ae8-49c9-8276-df9abd02ae31","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:57:25.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"9db16090-9ae8-49c9-8276-df9abd02ae31","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:57:25.606Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"9db16090-9ae8-49c9-8276-df9abd02ae31","podName":"counter-6m4r2-small-group-worker-nlscq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:25.606Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"9db16090-9ae8-49c9-8276-df9abd02ae31","podName":"counter-6m4r2-small-group-worker-nlscq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:57:25.607Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"9db16090-9ae8-49c9-8276-df9abd02ae31","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:57:25.607Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"9db16090-9ae8-49c9-8276-df9abd02ae31","seconds":300}
{"level":"info","ts":"2026-07-13T15:57:25.623Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"9858ed6b-3b9d-4bde-8f35-63b9ffb67b53","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:25.623Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"9858ed6b-3b9d-4bde-8f35-63b9ffb67b53","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:25.623Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"9858ed6b-3b9d-4bde-8f35-63b9ffb67b53","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:25.624Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"9858ed6b-3b9d-4bde-8f35-63b9ffb67b53"}
{"level":"info","ts":"2026-07-13T15:57:25.624Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"9858ed6b-3b9d-4bde-8f35-63b9ffb67b53","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:57:28.625Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a470fe97-e767-480b-ba2c-8870d029e1c8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:28.625Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a470fe97-e767-480b-ba2c-8870d029e1c8","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:28.625Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a470fe97-e767-480b-ba2c-8870d029e1c8","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:28.625Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a470fe97-e767-480b-ba2c-8870d029e1c8"}
{"level":"info","ts":"2026-07-13T15:57:28.625Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a470fe97-e767-480b-ba2c-8870d029e1c8","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:57:31.625Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"41c23f65-775b-422f-86ca-05c589aff802","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:31.625Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"41c23f65-775b-422f-86ca-05c589aff802","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:31.625Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"41c23f65-775b-422f-86ca-05c589aff802","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:31.625Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"41c23f65-775b-422f-86ca-05c589aff802"}
{"level":"info","ts":"2026-07-13T15:57:31.625Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"41c23f65-775b-422f-86ca-05c589aff802","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:57:34.626Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"b133558b-2110-4a61-977b-77eca8ca0082","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:34.626Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"b133558b-2110-4a61-977b-77eca8ca0082","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:34.626Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"b133558b-2110-4a61-977b-77eca8ca0082","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:34.626Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"b133558b-2110-4a61-977b-77eca8ca0082"}
{"level":"info","ts":"2026-07-13T15:57:34.626Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"b133558b-2110-4a61-977b-77eca8ca0082","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:57:34.940Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8a4593c5-5ff5-455f-bfc8-e08bb24a274e","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:57:34.940Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8a4593c5-5ff5-455f-bfc8-e08bb24a274e","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T15:57:34.940Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8a4593c5-5ff5-455f-bfc8-e08bb24a274e","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T15:57:34.940Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8a4593c5-5ff5-455f-bfc8-e08bb24a274e","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T15:57:37.627Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ef6b711b-c00b-4a64-bf7c-e874667dbaf4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:37.627Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ef6b711b-c00b-4a64-bf7c-e874667dbaf4","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:37.627Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ef6b711b-c00b-4a64-bf7c-e874667dbaf4","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:37.627Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ef6b711b-c00b-4a64-bf7c-e874667dbaf4"}
{"level":"info","ts":"2026-07-13T15:57:37.627Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ef6b711b-c00b-4a64-bf7c-e874667dbaf4","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:57:40.628Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"e2f4f8f8-2852-4e35-9b49-e55f74ae5ba3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:40.628Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"e2f4f8f8-2852-4e35-9b49-e55f74ae5ba3","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:40.628Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"e2f4f8f8-2852-4e35-9b49-e55f74ae5ba3","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:40.628Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"e2f4f8f8-2852-4e35-9b49-e55f74ae5ba3"}
{"level":"info","ts":"2026-07-13T15:57:40.628Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"e2f4f8f8-2852-4e35-9b49-e55f74ae5ba3","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:57:43.629Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a93503af-3881-46c9-965d-3dd445bf83a3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:43.629Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a93503af-3881-46c9-965d-3dd445bf83a3","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:43.629Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a93503af-3881-46c9-965d-3dd445bf83a3","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:43.629Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a93503af-3881-46c9-965d-3dd445bf83a3"}
{"level":"info","ts":"2026-07-13T15:57:43.629Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a93503af-3881-46c9-965d-3dd445bf83a3","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:57:46.630Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"4e146273-75e1-435e-a0ee-18ceb5ddf706","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:46.630Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"4e146273-75e1-435e-a0ee-18ceb5ddf706","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:46.631Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"4e146273-75e1-435e-a0ee-18ceb5ddf706","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:46.631Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"4e146273-75e1-435e-a0ee-18ceb5ddf706"}
{"level":"info","ts":"2026-07-13T15:57:46.631Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"4e146273-75e1-435e-a0ee-18ceb5ddf706","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:57:49.631Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"cdcfdc53-c24e-4b44-acc2-5832195eb749","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:49.631Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"cdcfdc53-c24e-4b44-acc2-5832195eb749","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:49.631Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"cdcfdc53-c24e-4b44-acc2-5832195eb749","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:49.632Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"cdcfdc53-c24e-4b44-acc2-5832195eb749"}
{"level":"info","ts":"2026-07-13T15:57:49.632Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"cdcfdc53-c24e-4b44-acc2-5832195eb749","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:57:50.539Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"3b53c330-ded0-4257-addb-dedc812795b1","rayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:50.539Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"3b53c330-ded0-4257-addb-dedc812795b1","pod":"counter-6m4r2-head-767d2","ip":"10.133.0.26"}
{"level":"info","ts":"2026-07-13T15:57:50.539Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"3b53c330-ded0-4257-addb-dedc812795b1","pod":"counter-6m4r2-small-group-worker-nlscq","ip":"10.134.0.30"}
{"level":"info","ts":"2026-07-13T15:57:50.549Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"3b53c330-ded0-4257-addb-dedc812795b1","certificate":"ray-head-cert-counter-6m4r2","dnsNames":["counter-6m4r2-head-svc","localhost","counter-6m4r2-head-svc.test-ns-88gnn.svc","counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local"],"ipAddresses":["10.133.0.26","10.134.0.30","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:57:50.559Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"3b53c330-ded0-4257-addb-dedc812795b1","certificate":"ray-worker-cert-counter-6m4r2","dnsNames":["counter-6m4r2-worker-svc","localhost","counter-6m4r2-small-group","counter-6m4r2-worker-svc.test-ns-88gnn.svc","counter-6m4r2-worker-svc.test-ns-88gnn.svc.cluster.local","*.counter-6m4r2-worker-svc.test-ns-88gnn.svc","*.counter-6m4r2-worker-svc.test-ns-88gnn.svc.cluster.local","*-worker-*.test-ns-88gnn.svc","*-worker-*.test-ns-88gnn.svc.cluster.local"],"ipAddresses":["10.133.0.26","10.134.0.30","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:57:50.559Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"3b53c330-ded0-4257-addb-dedc812795b1","rayCluster":"counter-6m4r2","podIPs":["10.133.0.26","10.134.0.30"]}
{"level":"info","ts":"2026-07-13T15:57:52.633Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"6c21c1f2-ba6a-4821-bd88-91d3f6b12408","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:52.633Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"6c21c1f2-ba6a-4821-bd88-91d3f6b12408","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:52.633Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"6c21c1f2-ba6a-4821-bd88-91d3f6b12408","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:52.633Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"6c21c1f2-ba6a-4821-bd88-91d3f6b12408"}
{"level":"info","ts":"2026-07-13T15:57:52.633Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"6c21c1f2-ba6a-4821-bd88-91d3f6b12408","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:57:55.634Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f1f5a42f-677d-4065-ad1e-e51d3f0f2cf0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:55.634Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f1f5a42f-677d-4065-ad1e-e51d3f0f2cf0","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:55.634Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f1f5a42f-677d-4065-ad1e-e51d3f0f2cf0","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:55.634Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f1f5a42f-677d-4065-ad1e-e51d3f0f2cf0"}
{"level":"info","ts":"2026-07-13T15:57:55.634Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f1f5a42f-677d-4065-ad1e-e51d3f0f2cf0","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:57:58.635Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"697ca024-9552-49a4-9530-f726768ac4c8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:57:58.635Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"697ca024-9552-49a4-9530-f726768ac4c8","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:57:58.635Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"697ca024-9552-49a4-9530-f726768ac4c8","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:57:58.636Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"697ca024-9552-49a4-9530-f726768ac4c8"}
{"level":"info","ts":"2026-07-13T15:57:58.636Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"697ca024-9552-49a4-9530-f726768ac4c8","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:01.636Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"4498846f-6808-4fa8-8fc4-cdd306866ea7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:01.636Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"4498846f-6808-4fa8-8fc4-cdd306866ea7","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:01.636Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"4498846f-6808-4fa8-8fc4-cdd306866ea7","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:01.637Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"4498846f-6808-4fa8-8fc4-cdd306866ea7"}
{"level":"info","ts":"2026-07-13T15:58:01.637Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"4498846f-6808-4fa8-8fc4-cdd306866ea7","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:04.637Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"4d0e8dd0-671f-45b8-8099-0701a39d82ff","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:04.637Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"4d0e8dd0-671f-45b8-8099-0701a39d82ff","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:04.637Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"4d0e8dd0-671f-45b8-8099-0701a39d82ff","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:04.637Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"4d0e8dd0-671f-45b8-8099-0701a39d82ff"}
{"level":"info","ts":"2026-07-13T15:58:04.637Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"4d0e8dd0-671f-45b8-8099-0701a39d82ff","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:07.638Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"7f499ed4-6dbe-4edd-af23-3b1764d12b1d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:07.638Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"7f499ed4-6dbe-4edd-af23-3b1764d12b1d","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:07.638Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"7f499ed4-6dbe-4edd-af23-3b1764d12b1d","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:07.638Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"7f499ed4-6dbe-4edd-af23-3b1764d12b1d"}
{"level":"info","ts":"2026-07-13T15:58:07.638Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"7f499ed4-6dbe-4edd-af23-3b1764d12b1d","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:10.639Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"dcb6ad3e-4307-4b6e-a4d5-4c501cce0e64","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:10.639Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"dcb6ad3e-4307-4b6e-a4d5-4c501cce0e64","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:10.639Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"dcb6ad3e-4307-4b6e-a4d5-4c501cce0e64","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:10.639Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"dcb6ad3e-4307-4b6e-a4d5-4c501cce0e64"}
{"level":"info","ts":"2026-07-13T15:58:10.639Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"dcb6ad3e-4307-4b6e-a4d5-4c501cce0e64","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:13.639Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a0d3c73d-cdc5-405b-af2b-a5a3fd574d77","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:13.639Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a0d3c73d-cdc5-405b-af2b-a5a3fd574d77","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:13.639Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a0d3c73d-cdc5-405b-af2b-a5a3fd574d77","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:13.640Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a0d3c73d-cdc5-405b-af2b-a5a3fd574d77"}
{"level":"info","ts":"2026-07-13T15:58:13.640Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a0d3c73d-cdc5-405b-af2b-a5a3fd574d77","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:16.640Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f0db4389-eb0f-491f-b3f1-fd43b48131ed","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:16.640Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f0db4389-eb0f-491f-b3f1-fd43b48131ed","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:16.640Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f0db4389-eb0f-491f-b3f1-fd43b48131ed","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:16.641Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f0db4389-eb0f-491f-b3f1-fd43b48131ed"}
{"level":"info","ts":"2026-07-13T15:58:16.641Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f0db4389-eb0f-491f-b3f1-fd43b48131ed","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:19.641Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"dbdf882d-1f16-4db2-aa9b-9e65b286dc28","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:19.641Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"dbdf882d-1f16-4db2-aa9b-9e65b286dc28","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:19.641Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"dbdf882d-1f16-4db2-aa9b-9e65b286dc28","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:19.642Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"dbdf882d-1f16-4db2-aa9b-9e65b286dc28"}
{"level":"info","ts":"2026-07-13T15:58:19.642Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"dbdf882d-1f16-4db2-aa9b-9e65b286dc28","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:22.642Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1474caf5-be9b-412b-853e-2d6b5a8854cc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:22.642Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1474caf5-be9b-412b-853e-2d6b5a8854cc","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:22.642Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1474caf5-be9b-412b-853e-2d6b5a8854cc","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:22.642Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1474caf5-be9b-412b-853e-2d6b5a8854cc"}
{"level":"info","ts":"2026-07-13T15:58:22.643Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1474caf5-be9b-412b-853e-2d6b5a8854cc","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:25.643Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"080ee8d2-096c-4cac-be47-a3c860c051b2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:25.643Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"080ee8d2-096c-4cac-be47-a3c860c051b2","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:25.643Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"080ee8d2-096c-4cac-be47-a3c860c051b2","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:25.643Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"080ee8d2-096c-4cac-be47-a3c860c051b2"}
{"level":"info","ts":"2026-07-13T15:58:25.643Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"080ee8d2-096c-4cac-be47-a3c860c051b2","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:28.643Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"da9282b2-9829-45fe-b57e-95fa857bd711","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:28.643Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"da9282b2-9829-45fe-b57e-95fa857bd711","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:28.643Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"da9282b2-9829-45fe-b57e-95fa857bd711","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:28.643Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"da9282b2-9829-45fe-b57e-95fa857bd711"}
{"level":"info","ts":"2026-07-13T15:58:28.644Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"da9282b2-9829-45fe-b57e-95fa857bd711","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:31.644Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1ce8f1ef-c19d-4c43-97bd-d8c0f38b75f9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:31.644Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1ce8f1ef-c19d-4c43-97bd-d8c0f38b75f9","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:31.644Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1ce8f1ef-c19d-4c43-97bd-d8c0f38b75f9","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:31.645Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1ce8f1ef-c19d-4c43-97bd-d8c0f38b75f9"}
{"level":"info","ts":"2026-07-13T15:58:31.645Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1ce8f1ef-c19d-4c43-97bd-d8c0f38b75f9","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:34.646Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"66616a31-6ff3-41cf-ba5f-69753ac436dd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:34.646Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"66616a31-6ff3-41cf-ba5f-69753ac436dd","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:34.646Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"66616a31-6ff3-41cf-ba5f-69753ac436dd","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:34.646Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"66616a31-6ff3-41cf-ba5f-69753ac436dd"}
{"level":"info","ts":"2026-07-13T15:58:34.646Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"66616a31-6ff3-41cf-ba5f-69753ac436dd","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:34.941Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"085b1542-bfb7-46f4-af0b-ae44c8b644e1","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:58:34.941Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"085b1542-bfb7-46f4-af0b-ae44c8b644e1","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T15:58:34.941Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"085b1542-bfb7-46f4-af0b-ae44c8b644e1","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T15:58:34.942Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"085b1542-bfb7-46f4-af0b-ae44c8b644e1","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T15:58:37.647Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ca541b57-ca24-48db-9e47-ec20c3d4e9d5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:37.647Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ca541b57-ca24-48db-9e47-ec20c3d4e9d5","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:37.647Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ca541b57-ca24-48db-9e47-ec20c3d4e9d5","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:37.647Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ca541b57-ca24-48db-9e47-ec20c3d4e9d5"}
{"level":"info","ts":"2026-07-13T15:58:37.647Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ca541b57-ca24-48db-9e47-ec20c3d4e9d5","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:38.867Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"62b0f429-5f0f-4916-b967-8ea467f883ba"}
{"level":"info","ts":"2026-07-13T15:58:38.867Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"62b0f429-5f0f-4916-b967-8ea467f883ba"}
{"level":"info","ts":"2026-07-13T15:58:38.867Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"62b0f429-5f0f-4916-b967-8ea467f883ba","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:58:38.867Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"62b0f429-5f0f-4916-b967-8ea467f883ba","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:58:38.867Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"62b0f429-5f0f-4916-b967-8ea467f883ba","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:58:38.867Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"62b0f429-5f0f-4916-b967-8ea467f883ba","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:58:38.867Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"62b0f429-5f0f-4916-b967-8ea467f883ba","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:58:38.867Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"62b0f429-5f0f-4916-b967-8ea467f883ba","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:58:38.867Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"62b0f429-5f0f-4916-b967-8ea467f883ba","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:58:38.867Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"62b0f429-5f0f-4916-b967-8ea467f883ba","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:58:38.867Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"62b0f429-5f0f-4916-b967-8ea467f883ba","podName":"counter-6m4r2-head-767d2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:58:38.867Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"62b0f429-5f0f-4916-b967-8ea467f883ba","podName":"counter-6m4r2-head-767d2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:58:38.867Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"62b0f429-5f0f-4916-b967-8ea467f883ba","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:58:38.867Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"62b0f429-5f0f-4916-b967-8ea467f883ba","seconds":300}
{"level":"info","ts":"2026-07-13T15:58:39.873Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"90a0535b-2f7f-4c63-b0d3-bbde55b17b04"}
{"level":"info","ts":"2026-07-13T15:58:39.873Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"90a0535b-2f7f-4c63-b0d3-bbde55b17b04"}
{"level":"info","ts":"2026-07-13T15:58:39.873Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"90a0535b-2f7f-4c63-b0d3-bbde55b17b04","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:58:39.873Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"90a0535b-2f7f-4c63-b0d3-bbde55b17b04","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:58:39.873Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"90a0535b-2f7f-4c63-b0d3-bbde55b17b04","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:58:39.873Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"90a0535b-2f7f-4c63-b0d3-bbde55b17b04","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:58:39.873Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"90a0535b-2f7f-4c63-b0d3-bbde55b17b04","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:58:39.873Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"90a0535b-2f7f-4c63-b0d3-bbde55b17b04","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:58:39.873Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"90a0535b-2f7f-4c63-b0d3-bbde55b17b04","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:58:39.873Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"90a0535b-2f7f-4c63-b0d3-bbde55b17b04","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:58:39.873Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"90a0535b-2f7f-4c63-b0d3-bbde55b17b04","podName":"counter-6m4r2-head-767d2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:57:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:58:39.873Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"90a0535b-2f7f-4c63-b0d3-bbde55b17b04","podName":"counter-6m4r2-head-767d2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:57:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:58:39.873Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"90a0535b-2f7f-4c63-b0d3-bbde55b17b04","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:58:39.873Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"90a0535b-2f7f-4c63-b0d3-bbde55b17b04","name":"counter-6m4r2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:57:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:58:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:58:39.886Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"14850857-9c5f-47a0-9a46-f0379d749045","rayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:39.886Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"1dd3ba8f-4253-460f-a710-904f5636e330","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:39.886Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"14850857-9c5f-47a0-9a46-f0379d749045","pod":"counter-6m4r2-head-767d2","ip":"10.133.0.26"}
{"level":"info","ts":"2026-07-13T15:58:39.886Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"14850857-9c5f-47a0-9a46-f0379d749045","pod":"counter-6m4r2-small-group-worker-nlscq","ip":"10.134.0.30"}
{"level":"info","ts":"2026-07-13T15:58:39.886Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"9acbb02d-20cf-449f-9562-d0756ef60c71","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:39.886Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"9acbb02d-20cf-449f-9562-d0756ef60c71","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:39.886Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"14850857-9c5f-47a0-9a46-f0379d749045","rayCluster":"counter-6m4r2","podIPs":["10.133.0.26","10.134.0.30"]}
{"level":"info","ts":"2026-07-13T15:58:39.886Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"9acbb02d-20cf-449f-9562-d0756ef60c71","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:39.886Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"9acbb02d-20cf-449f-9562-d0756ef60c71"}
{"level":"info","ts":"2026-07-13T15:58:39.886Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"9acbb02d-20cf-449f-9562-d0756ef60c71","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:39.901Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"1dd3ba8f-4253-460f-a710-904f5636e330","name":"counter-6m4r2-head"}
{"level":"info","ts":"2026-07-13T15:58:39.912Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"1dd3ba8f-4253-460f-a710-904f5636e330","name":"counter-6m4r2-workers"}
{"level":"info","ts":"2026-07-13T15:58:39.912Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"1dd3ba8f-4253-460f-a710-904f5636e330","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:40.648Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"494ebf31-84fa-4a09-a54a-ed2c66f18a65","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:40.648Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"494ebf31-84fa-4a09-a54a-ed2c66f18a65","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:40.648Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"494ebf31-84fa-4a09-a54a-ed2c66f18a65","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:40.648Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"494ebf31-84fa-4a09-a54a-ed2c66f18a65"}
{"level":"info","ts":"2026-07-13T15:58:40.648Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"494ebf31-84fa-4a09-a54a-ed2c66f18a65","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:41.886Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bab5b10b-1f6f-4a60-b986-bd33d0c6a8e0"}
{"level":"info","ts":"2026-07-13T15:58:41.886Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bab5b10b-1f6f-4a60-b986-bd33d0c6a8e0"}
{"level":"info","ts":"2026-07-13T15:58:41.886Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bab5b10b-1f6f-4a60-b986-bd33d0c6a8e0","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:58:41.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bab5b10b-1f6f-4a60-b986-bd33d0c6a8e0","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:58:41.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bab5b10b-1f6f-4a60-b986-bd33d0c6a8e0","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:58:41.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bab5b10b-1f6f-4a60-b986-bd33d0c6a8e0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:58:41.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bab5b10b-1f6f-4a60-b986-bd33d0c6a8e0","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:58:41.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bab5b10b-1f6f-4a60-b986-bd33d0c6a8e0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:58:41.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bab5b10b-1f6f-4a60-b986-bd33d0c6a8e0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:58:41.886Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bab5b10b-1f6f-4a60-b986-bd33d0c6a8e0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:58:41.886Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bab5b10b-1f6f-4a60-b986-bd33d0c6a8e0","podName":"counter-6m4r2-head-767d2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:57:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:58:41.886Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bab5b10b-1f6f-4a60-b986-bd33d0c6a8e0","podName":"counter-6m4r2-head-767d2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:57:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:58:41.886Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bab5b10b-1f6f-4a60-b986-bd33d0c6a8e0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:58:41.886Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"bab5b10b-1f6f-4a60-b986-bd33d0c6a8e0","seconds":300}
{"level":"info","ts":"2026-07-13T15:58:43.648Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f4f1b616-e4a3-4021-baa0-2dcc2517c91d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:43.649Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f4f1b616-e4a3-4021-baa0-2dcc2517c91d","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:43.649Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f4f1b616-e4a3-4021-baa0-2dcc2517c91d","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:43.649Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f4f1b616-e4a3-4021-baa0-2dcc2517c91d"}
{"level":"info","ts":"2026-07-13T15:58:43.649Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f4f1b616-e4a3-4021-baa0-2dcc2517c91d","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:46.649Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"615c893d-0be0-4caf-994a-e5abb4410bd8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:46.649Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"615c893d-0be0-4caf-994a-e5abb4410bd8","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:46.649Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"615c893d-0be0-4caf-994a-e5abb4410bd8","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:46.649Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"615c893d-0be0-4caf-994a-e5abb4410bd8"}
{"level":"info","ts":"2026-07-13T15:58:46.649Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"615c893d-0be0-4caf-994a-e5abb4410bd8","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:49.151Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"7727534d-ca73-4ce5-9677-a5fecdb0d2be"}
{"level":"info","ts":"2026-07-13T15:58:49.151Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"7727534d-ca73-4ce5-9677-a5fecdb0d2be"}
{"level":"info","ts":"2026-07-13T15:58:49.151Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"7727534d-ca73-4ce5-9677-a5fecdb0d2be","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:58:49.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"7727534d-ca73-4ce5-9677-a5fecdb0d2be","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:58:49.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"7727534d-ca73-4ce5-9677-a5fecdb0d2be","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:58:49.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"7727534d-ca73-4ce5-9677-a5fecdb0d2be","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:58:49.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"7727534d-ca73-4ce5-9677-a5fecdb0d2be","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:58:49.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"7727534d-ca73-4ce5-9677-a5fecdb0d2be","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:58:49.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"7727534d-ca73-4ce5-9677-a5fecdb0d2be","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:58:49.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"7727534d-ca73-4ce5-9677-a5fecdb0d2be","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:58:49.151Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"7727534d-ca73-4ce5-9677-a5fecdb0d2be","podName":"counter-6m4r2-head-767d2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:57:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:58:49.151Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"7727534d-ca73-4ce5-9677-a5fecdb0d2be","podName":"counter-6m4r2-head-767d2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:57:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:58:49.151Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"7727534d-ca73-4ce5-9677-a5fecdb0d2be","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:58:49.151Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"7727534d-ca73-4ce5-9677-a5fecdb0d2be","seconds":300}
{"level":"info","ts":"2026-07-13T15:58:49.650Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"30f4592d-941d-4b94-8b3b-2a872426a35b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:49.650Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"30f4592d-941d-4b94-8b3b-2a872426a35b","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:49.650Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"30f4592d-941d-4b94-8b3b-2a872426a35b","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:49.650Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"30f4592d-941d-4b94-8b3b-2a872426a35b"}
{"level":"info","ts":"2026-07-13T15:58:49.650Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"30f4592d-941d-4b94-8b3b-2a872426a35b","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:50.153Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6c51b63b-4717-4126-8e4a-d44c12d00469"}
{"level":"info","ts":"2026-07-13T15:58:50.153Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6c51b63b-4717-4126-8e4a-d44c12d00469"}
{"level":"info","ts":"2026-07-13T15:58:50.153Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6c51b63b-4717-4126-8e4a-d44c12d00469","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:58:50.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6c51b63b-4717-4126-8e4a-d44c12d00469","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:58:50.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6c51b63b-4717-4126-8e4a-d44c12d00469","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:58:50.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6c51b63b-4717-4126-8e4a-d44c12d00469","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:58:50.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6c51b63b-4717-4126-8e4a-d44c12d00469","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:58:50.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6c51b63b-4717-4126-8e4a-d44c12d00469","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:58:50.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6c51b63b-4717-4126-8e4a-d44c12d00469","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:58:50.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6c51b63b-4717-4126-8e4a-d44c12d00469","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:58:50.221Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6c51b63b-4717-4126-8e4a-d44c12d00469","podName":"counter-6m4r2-head-767d2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:57:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:58:50.221Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6c51b63b-4717-4126-8e4a-d44c12d00469","podName":"counter-6m4r2-head-767d2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:57:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:58:50.222Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6c51b63b-4717-4126-8e4a-d44c12d00469","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:58:50.222Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6c51b63b-4717-4126-8e4a-d44c12d00469","seconds":300}
{"level":"info","ts":"2026-07-13T15:58:50.560Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"8a68e9b5-5986-4d9b-b89a-23eb7143e418","rayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:50.560Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"8a68e9b5-5986-4d9b-b89a-23eb7143e418","pod":"counter-6m4r2-head-767d2","ip":"10.133.0.26"}
{"level":"info","ts":"2026-07-13T15:58:50.560Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"8a68e9b5-5986-4d9b-b89a-23eb7143e418","pod":"counter-6m4r2-small-group-worker-nlscq","ip":"10.134.0.30"}
{"level":"info","ts":"2026-07-13T15:58:50.560Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"8a68e9b5-5986-4d9b-b89a-23eb7143e418","rayCluster":"counter-6m4r2","podIPs":["10.133.0.26","10.134.0.30"]}
{"level":"info","ts":"2026-07-13T15:58:50.906Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"127aaeb3-e641-4f5a-9e97-ff924697553f"}
{"level":"info","ts":"2026-07-13T15:58:50.906Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"127aaeb3-e641-4f5a-9e97-ff924697553f"}
{"level":"info","ts":"2026-07-13T15:58:50.906Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"127aaeb3-e641-4f5a-9e97-ff924697553f","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:58:50.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"127aaeb3-e641-4f5a-9e97-ff924697553f","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:58:50.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"127aaeb3-e641-4f5a-9e97-ff924697553f","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:58:50.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"127aaeb3-e641-4f5a-9e97-ff924697553f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:58:50.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"127aaeb3-e641-4f5a-9e97-ff924697553f","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:58:50.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"127aaeb3-e641-4f5a-9e97-ff924697553f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:58:50.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"127aaeb3-e641-4f5a-9e97-ff924697553f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:58:50.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"127aaeb3-e641-4f5a-9e97-ff924697553f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:58:50.907Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"127aaeb3-e641-4f5a-9e97-ff924697553f","podName":"counter-6m4r2-small-group-worker-nlscq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:58:50.907Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"127aaeb3-e641-4f5a-9e97-ff924697553f","podName":"counter-6m4r2-small-group-worker-nlscq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:58:50.907Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"127aaeb3-e641-4f5a-9e97-ff924697553f","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:58:50.907Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"127aaeb3-e641-4f5a-9e97-ff924697553f","name":"counter-6m4r2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:58:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:58:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:58:50.918Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"f9fa3aad-f4a6-4f3e-b7b6-e9201b408624","rayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:50.918Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"a02e0db2-b056-4d6a-b848-9cba09caf63c","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:50.918Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"f9fa3aad-f4a6-4f3e-b7b6-e9201b408624","pod":"counter-6m4r2-head-767d2","ip":"10.133.0.26"}
{"level":"info","ts":"2026-07-13T15:58:50.918Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"f9fa3aad-f4a6-4f3e-b7b6-e9201b408624","pod":"counter-6m4r2-small-group-worker-nlscq","ip":"10.134.0.30"}
{"level":"info","ts":"2026-07-13T15:58:50.919Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"d7e8f00e-64cc-4e1c-a4ab-2e662c01a7cc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:50.919Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"d7e8f00e-64cc-4e1c-a4ab-2e662c01a7cc","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:50.919Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"d7e8f00e-64cc-4e1c-a4ab-2e662c01a7cc","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:50.919Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"f9fa3aad-f4a6-4f3e-b7b6-e9201b408624","rayCluster":"counter-6m4r2","podIPs":["10.133.0.26","10.134.0.30"]}
{"level":"info","ts":"2026-07-13T15:58:50.919Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"d7e8f00e-64cc-4e1c-a4ab-2e662c01a7cc"}
{"level":"info","ts":"2026-07-13T15:58:50.919Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"d7e8f00e-64cc-4e1c-a4ab-2e662c01a7cc","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:50.928Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"a02e0db2-b056-4d6a-b848-9cba09caf63c","name":"counter-6m4r2-head"}
{"level":"info","ts":"2026-07-13T15:58:50.937Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"a02e0db2-b056-4d6a-b848-9cba09caf63c","name":"counter-6m4r2-workers"}
{"level":"info","ts":"2026-07-13T15:58:50.937Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"a02e0db2-b056-4d6a-b848-9cba09caf63c","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:52.651Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"3a280b26-0feb-4310-9b14-8fbcdc4f7abf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:52.651Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"3a280b26-0feb-4310-9b14-8fbcdc4f7abf","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:52.651Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"3a280b26-0feb-4310-9b14-8fbcdc4f7abf","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:52.652Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"3a280b26-0feb-4310-9b14-8fbcdc4f7abf"}
{"level":"info","ts":"2026-07-13T15:58:52.652Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"3a280b26-0feb-4310-9b14-8fbcdc4f7abf","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:52.918Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"080fb050-8264-4616-af05-ac3e66c7dfa1"}
{"level":"info","ts":"2026-07-13T15:58:52.918Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"080fb050-8264-4616-af05-ac3e66c7dfa1"}
{"level":"info","ts":"2026-07-13T15:58:52.918Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"080fb050-8264-4616-af05-ac3e66c7dfa1","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:58:52.919Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"080fb050-8264-4616-af05-ac3e66c7dfa1","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:58:52.919Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"080fb050-8264-4616-af05-ac3e66c7dfa1","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:58:52.919Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"080fb050-8264-4616-af05-ac3e66c7dfa1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:58:52.919Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"080fb050-8264-4616-af05-ac3e66c7dfa1","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:58:52.919Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"080fb050-8264-4616-af05-ac3e66c7dfa1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:58:52.919Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"080fb050-8264-4616-af05-ac3e66c7dfa1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:58:52.919Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"080fb050-8264-4616-af05-ac3e66c7dfa1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:58:52.919Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"080fb050-8264-4616-af05-ac3e66c7dfa1","podName":"counter-6m4r2-small-group-worker-nlscq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:58:52.919Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"080fb050-8264-4616-af05-ac3e66c7dfa1","podName":"counter-6m4r2-small-group-worker-nlscq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:58:52.919Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"080fb050-8264-4616-af05-ac3e66c7dfa1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:58:52.919Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"080fb050-8264-4616-af05-ac3e66c7dfa1","seconds":300}
{"level":"info","ts":"2026-07-13T15:58:55.652Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"05350122-0653-41a1-bb00-24566981db17","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:55.652Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"05350122-0653-41a1-bb00-24566981db17","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:55.652Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"05350122-0653-41a1-bb00-24566981db17","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:55.652Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"05350122-0653-41a1-bb00-24566981db17"}
{"level":"info","ts":"2026-07-13T15:58:55.652Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"05350122-0653-41a1-bb00-24566981db17","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:58:58.653Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a15f2783-3b63-40c2-b454-24e57cc14c9a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:58:58.653Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a15f2783-3b63-40c2-b454-24e57cc14c9a","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:58:58.653Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a15f2783-3b63-40c2-b454-24e57cc14c9a","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:58:58.653Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a15f2783-3b63-40c2-b454-24e57cc14c9a"}
{"level":"info","ts":"2026-07-13T15:58:58.653Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"a15f2783-3b63-40c2-b454-24e57cc14c9a","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:59:00.198Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"46e939de-7509-41fa-85db-60706d5ff645"}
{"level":"info","ts":"2026-07-13T15:59:00.198Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"46e939de-7509-41fa-85db-60706d5ff645"}
{"level":"info","ts":"2026-07-13T15:59:00.198Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"46e939de-7509-41fa-85db-60706d5ff645","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:00.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"46e939de-7509-41fa-85db-60706d5ff645","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:00.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"46e939de-7509-41fa-85db-60706d5ff645","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:00.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"46e939de-7509-41fa-85db-60706d5ff645","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:00.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"46e939de-7509-41fa-85db-60706d5ff645","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:00.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"46e939de-7509-41fa-85db-60706d5ff645","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:00.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"46e939de-7509-41fa-85db-60706d5ff645","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:00.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"46e939de-7509-41fa-85db-60706d5ff645","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:00.198Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"46e939de-7509-41fa-85db-60706d5ff645","podName":"counter-6m4r2-small-group-worker-nlscq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:00.198Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"46e939de-7509-41fa-85db-60706d5ff645","podName":"counter-6m4r2-small-group-worker-nlscq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:00.198Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"46e939de-7509-41fa-85db-60706d5ff645","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:59:00.198Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"46e939de-7509-41fa-85db-60706d5ff645","seconds":300}
{"level":"info","ts":"2026-07-13T15:59:01.208Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b1b580b3-2d86-4f3e-97c8-f829626c55eb"}
{"level":"info","ts":"2026-07-13T15:59:01.208Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b1b580b3-2d86-4f3e-97c8-f829626c55eb"}
{"level":"info","ts":"2026-07-13T15:59:01.208Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b1b580b3-2d86-4f3e-97c8-f829626c55eb","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:01.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b1b580b3-2d86-4f3e-97c8-f829626c55eb","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:01.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b1b580b3-2d86-4f3e-97c8-f829626c55eb","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:01.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b1b580b3-2d86-4f3e-97c8-f829626c55eb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:01.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b1b580b3-2d86-4f3e-97c8-f829626c55eb","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:01.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b1b580b3-2d86-4f3e-97c8-f829626c55eb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:01.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b1b580b3-2d86-4f3e-97c8-f829626c55eb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:01.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b1b580b3-2d86-4f3e-97c8-f829626c55eb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:01.209Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b1b580b3-2d86-4f3e-97c8-f829626c55eb","podName":"counter-6m4r2-small-group-worker-nlscq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:57:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:59:01.209Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b1b580b3-2d86-4f3e-97c8-f829626c55eb","podName":"counter-6m4r2-small-group-worker-nlscq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:57:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:59:01.209Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b1b580b3-2d86-4f3e-97c8-f829626c55eb","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:59:01.209Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"b1b580b3-2d86-4f3e-97c8-f829626c55eb","name":"counter-6m4r2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:58:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:59:01.223Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"15e48110-f1d1-475f-957b-7b10b5958be7","rayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:01.223Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"156fdd62-b11c-4bee-854f-11b36e5add31","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:01.223Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"156fdd62-b11c-4bee-854f-11b36e5add31","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:01.223Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"e7034520-897c-4a9e-ad44-434953e38a00","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:01.223Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"156fdd62-b11c-4bee-854f-11b36e5add31","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:01.223Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"15e48110-f1d1-475f-957b-7b10b5958be7","pod":"counter-6m4r2-head-767d2","ip":"10.133.0.26"}
{"level":"info","ts":"2026-07-13T15:59:01.223Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"15e48110-f1d1-475f-957b-7b10b5958be7","pod":"counter-6m4r2-small-group-worker-nlscq","ip":"10.134.0.30"}
{"level":"info","ts":"2026-07-13T15:59:01.223Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"15e48110-f1d1-475f-957b-7b10b5958be7","rayCluster":"counter-6m4r2","podIPs":["10.133.0.26","10.134.0.30"]}
{"level":"info","ts":"2026-07-13T15:59:01.223Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"156fdd62-b11c-4bee-854f-11b36e5add31"}
{"level":"info","ts":"2026-07-13T15:59:01.223Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"156fdd62-b11c-4bee-854f-11b36e5add31","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:59:01.236Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"e7034520-897c-4a9e-ad44-434953e38a00","name":"counter-6m4r2-head"}
{"level":"info","ts":"2026-07-13T15:59:01.245Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"e7034520-897c-4a9e-ad44-434953e38a00","name":"counter-6m4r2-workers"}
{"level":"info","ts":"2026-07-13T15:59:01.245Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"e7034520-897c-4a9e-ad44-434953e38a00","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:01.654Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"0d84e3b3-f4f0-4eac-b247-327f4b10ec5e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:01.654Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"0d84e3b3-f4f0-4eac-b247-327f4b10ec5e","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:01.654Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"0d84e3b3-f4f0-4eac-b247-327f4b10ec5e","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:01.654Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"0d84e3b3-f4f0-4eac-b247-327f4b10ec5e"}
{"level":"info","ts":"2026-07-13T15:59:01.654Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"0d84e3b3-f4f0-4eac-b247-327f4b10ec5e","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:59:03.223Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9afe82b-8f99-4d47-9c31-6a38479f2d8f"}
{"level":"info","ts":"2026-07-13T15:59:03.223Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9afe82b-8f99-4d47-9c31-6a38479f2d8f"}
{"level":"info","ts":"2026-07-13T15:59:03.223Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9afe82b-8f99-4d47-9c31-6a38479f2d8f","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:03.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9afe82b-8f99-4d47-9c31-6a38479f2d8f","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:03.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9afe82b-8f99-4d47-9c31-6a38479f2d8f","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:03.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9afe82b-8f99-4d47-9c31-6a38479f2d8f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:03.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9afe82b-8f99-4d47-9c31-6a38479f2d8f","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:03.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9afe82b-8f99-4d47-9c31-6a38479f2d8f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:03.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9afe82b-8f99-4d47-9c31-6a38479f2d8f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:03.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9afe82b-8f99-4d47-9c31-6a38479f2d8f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:03.224Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9afe82b-8f99-4d47-9c31-6a38479f2d8f","podName":"counter-6m4r2-small-group-worker-nlscq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:57:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:59:03.224Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9afe82b-8f99-4d47-9c31-6a38479f2d8f","podName":"counter-6m4r2-small-group-worker-nlscq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:57:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:59:03.224Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9afe82b-8f99-4d47-9c31-6a38479f2d8f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:59:03.224Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"a9afe82b-8f99-4d47-9c31-6a38479f2d8f","seconds":300}
{"level":"info","ts":"2026-07-13T15:59:04.655Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"9651d294-8642-43e8-bd65-786986bdfb44","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:04.655Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"9651d294-8642-43e8-bd65-786986bdfb44","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:04.655Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"9651d294-8642-43e8-bd65-786986bdfb44","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:04.655Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"9651d294-8642-43e8-bd65-786986bdfb44"}
{"level":"info","ts":"2026-07-13T15:59:04.655Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"9651d294-8642-43e8-bd65-786986bdfb44","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:59:07.656Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"9ed91209-f2b3-4ec3-9504-a71a1381c4ac","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:07.656Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"9ed91209-f2b3-4ec3-9504-a71a1381c4ac","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:07.656Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"9ed91209-f2b3-4ec3-9504-a71a1381c4ac","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:07.656Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"9ed91209-f2b3-4ec3-9504-a71a1381c4ac"}
{"level":"info","ts":"2026-07-13T15:59:07.656Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"9ed91209-f2b3-4ec3-9504-a71a1381c4ac","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:59:10.657Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f5a8ff37-780a-4c53-970d-193a598003e8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:10.657Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f5a8ff37-780a-4c53-970d-193a598003e8","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:10.657Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f5a8ff37-780a-4c53-970d-193a598003e8","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:10.657Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f5a8ff37-780a-4c53-970d-193a598003e8"}
{"level":"info","ts":"2026-07-13T15:59:10.657Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f5a8ff37-780a-4c53-970d-193a598003e8","RayCluster":"counter-6m4r2","State":""}
{"level":"info","ts":"2026-07-13T15:59:12.227Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6889fb79-9a56-4960-b58f-2e53669a4a66"}
{"level":"info","ts":"2026-07-13T15:59:12.227Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6889fb79-9a56-4960-b58f-2e53669a4a66"}
{"level":"info","ts":"2026-07-13T15:59:12.227Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6889fb79-9a56-4960-b58f-2e53669a4a66","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:12.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6889fb79-9a56-4960-b58f-2e53669a4a66","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:12.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6889fb79-9a56-4960-b58f-2e53669a4a66","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:12.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6889fb79-9a56-4960-b58f-2e53669a4a66","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:12.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6889fb79-9a56-4960-b58f-2e53669a4a66","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:12.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6889fb79-9a56-4960-b58f-2e53669a4a66","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:12.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6889fb79-9a56-4960-b58f-2e53669a4a66","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:12.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6889fb79-9a56-4960-b58f-2e53669a4a66","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:12.227Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6889fb79-9a56-4960-b58f-2e53669a4a66","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-13T15:59:12.227Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"6889fb79-9a56-4960-b58f-2e53669a4a66","name":"counter-6m4r2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:12Z","stateTransitionTimes":{"ready":"2026-07-13T15:59:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:59:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:59:12.240Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"bc05c2cd-49a2-423a-9a91-92f8887a0070","rayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:12.240Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"ff6af80e-2686-47f5-b356-c9d88934d02b","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:12.240Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"bc05c2cd-49a2-423a-9a91-92f8887a0070","pod":"counter-6m4r2-head-767d2","ip":"10.133.0.26"}
{"level":"info","ts":"2026-07-13T15:59:12.240Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"bc05c2cd-49a2-423a-9a91-92f8887a0070","pod":"counter-6m4r2-small-group-worker-nlscq","ip":"10.134.0.30"}
{"level":"info","ts":"2026-07-13T15:59:12.240Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"bc05c2cd-49a2-423a-9a91-92f8887a0070","rayCluster":"counter-6m4r2","podIPs":["10.133.0.26","10.134.0.30"]}
{"level":"info","ts":"2026-07-13T15:59:12.240Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ab7b1182-cedb-4600-8d14-7f83a9d8c483","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:12.240Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ab7b1182-cedb-4600-8d14-7f83a9d8c483","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:12.240Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ab7b1182-cedb-4600-8d14-7f83a9d8c483","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:12.240Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ab7b1182-cedb-4600-8d14-7f83a9d8c483"}
{"level":"info","ts":"2026-07-13T15:59:12.241Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ab7b1182-cedb-4600-8d14-7f83a9d8c483","head service name":"counter-6m4r2-head-svc","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:59:12.241Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ab7b1182-cedb-4600-8d14-7f83a9d8c483","head service URL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-13T15:59:12.241Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ab7b1182-cedb-4600-8d14-7f83a9d8c483"}
{"level":"info","ts":"2026-07-13T15:59:12.241Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ab7b1182-cedb-4600-8d14-7f83a9d8c483","command":["if","!","ray","job","status","--address","http://counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","counter-kjwgl",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","--no-wait","--runtime-env-json","\"{\\\"env_vars\\\":{\\\"counter_name\\\":\\\"test_counter\\\"}}\"","--submission-id","counter-kjwgl","--","python /home/ray/jobs/counter.py",";","fi",";","ray","job","logs","--address","http://counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","--follow","counter-kjwgl"]}
{"level":"info","ts":"2026-07-13T15:59:12.253Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ab7b1182-cedb-4600-8d14-7f83a9d8c483","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-07-13T15:59:12.253Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ab7b1182-cedb-4600-8d14-7f83a9d8c483","SubmissionMode":"K8sJobMode","RayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:12.253Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ab7b1182-cedb-4600-8d14-7f83a9d8c483","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","dashboardURL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:59:12.253Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ab7b1182-cedb-4600-8d14-7f83a9d8c483","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T15:59:12.256Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"ff6af80e-2686-47f5-b356-c9d88934d02b","name":"counter-6m4r2-head"}
{"level":"info","ts":"2026-07-13T15:59:12.265Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"92618473-24ac-46bd-a3d7-9e8c799c14a0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:12.265Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"92618473-24ac-46bd-a3d7-9e8c799c14a0","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:12.265Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"92618473-24ac-46bd-a3d7-9e8c799c14a0","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:12.265Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"92618473-24ac-46bd-a3d7-9e8c799c14a0"}
{"level":"info","ts":"2026-07-13T15:59:12.265Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"92618473-24ac-46bd-a3d7-9e8c799c14a0","head service name":"counter-6m4r2-head-svc","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:59:12.265Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"92618473-24ac-46bd-a3d7-9e8c799c14a0","head service URL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-13T15:59:12.265Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"92618473-24ac-46bd-a3d7-9e8c799c14a0","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-07-13T15:59:12.265Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"92618473-24ac-46bd-a3d7-9e8c799c14a0","SubmissionMode":"K8sJobMode","RayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:12.265Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"92618473-24ac-46bd-a3d7-9e8c799c14a0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","dashboardURL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:59:12.265Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"92618473-24ac-46bd-a3d7-9e8c799c14a0","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T15:59:12.267Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"ff6af80e-2686-47f5-b356-c9d88934d02b","name":"counter-6m4r2-workers"}
{"level":"info","ts":"2026-07-13T15:59:12.267Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"ff6af80e-2686-47f5-b356-c9d88934d02b","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:12.272Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"92618473-24ac-46bd-a3d7-9e8c799c14a0","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-13T15:59:12.272Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"92618473-24ac-46bd-a3d7-9e8c799c14a0"}
{"level":"error","ts":"2026-07-13T15:59:12.272Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"92618473-24ac-46bd-a3d7-9e8c799c14a0","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:12.272Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"5dbbd545-7f0b-40f3-bb76-b40bb0842003","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:12.272Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"5dbbd545-7f0b-40f3-bb76-b40bb0842003","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:12.272Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"5dbbd545-7f0b-40f3-bb76-b40bb0842003","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:12.272Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"5dbbd545-7f0b-40f3-bb76-b40bb0842003"}
{"level":"error","ts":"2026-07-13T15:59:12.326Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"5dbbd545-7f0b-40f3-bb76-b40bb0842003","JobId":"counter-kjwgl","error":"Job counter-kjwgl does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:12.326Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"5dbbd545-7f0b-40f3-bb76-b40bb0842003"}
{"level":"error","ts":"2026-07-13T15:59:12.326Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"5dbbd545-7f0b-40f3-bb76-b40bb0842003","error":"Job counter-kjwgl does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:12.326Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"b4b23026-15cc-4960-b2f9-15a3df455ab0","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:12.326Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"b4b23026-15cc-4960-b2f9-15a3df455ab0","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:12.326Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"b4b23026-15cc-4960-b2f9-15a3df455ab0","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:12.326Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"b4b23026-15cc-4960-b2f9-15a3df455ab0"}
{"level":"error","ts":"2026-07-13T15:59:12.331Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"b4b23026-15cc-4960-b2f9-15a3df455ab0","JobId":"counter-kjwgl","error":"Job counter-kjwgl does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:12.331Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"b4b23026-15cc-4960-b2f9-15a3df455ab0"}
{"level":"error","ts":"2026-07-13T15:59:12.331Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"b4b23026-15cc-4960-b2f9-15a3df455ab0","error":"Job counter-kjwgl does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:12.337Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"59bfc6f8-cb43-4dea-9544-1468c496059d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:12.337Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"59bfc6f8-cb43-4dea-9544-1468c496059d","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:12.337Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"59bfc6f8-cb43-4dea-9544-1468c496059d","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:12.337Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"59bfc6f8-cb43-4dea-9544-1468c496059d"}
{"level":"error","ts":"2026-07-13T15:59:12.341Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"59bfc6f8-cb43-4dea-9544-1468c496059d","JobId":"counter-kjwgl","error":"Job counter-kjwgl does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:12.341Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"59bfc6f8-cb43-4dea-9544-1468c496059d"}
{"level":"error","ts":"2026-07-13T15:59:12.341Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"59bfc6f8-cb43-4dea-9544-1468c496059d","error":"Job counter-kjwgl does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:12.381Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"60db45dc-18f9-4f9d-97a2-a16b521d92fe","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:12.381Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"60db45dc-18f9-4f9d-97a2-a16b521d92fe","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:12.381Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"60db45dc-18f9-4f9d-97a2-a16b521d92fe","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:12.381Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"60db45dc-18f9-4f9d-97a2-a16b521d92fe"}
{"level":"error","ts":"2026-07-13T15:59:12.385Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"60db45dc-18f9-4f9d-97a2-a16b521d92fe","JobId":"counter-kjwgl","error":"Job counter-kjwgl does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:12.385Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"60db45dc-18f9-4f9d-97a2-a16b521d92fe"}
{"level":"error","ts":"2026-07-13T15:59:12.385Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"60db45dc-18f9-4f9d-97a2-a16b521d92fe","error":"Job counter-kjwgl does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:12.466Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"6efcf9f7-3559-4c25-8a66-dbe43dd2eece","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:12.466Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"6efcf9f7-3559-4c25-8a66-dbe43dd2eece","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:12.466Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"6efcf9f7-3559-4c25-8a66-dbe43dd2eece","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:12.466Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"6efcf9f7-3559-4c25-8a66-dbe43dd2eece"}
{"level":"error","ts":"2026-07-13T15:59:12.470Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"6efcf9f7-3559-4c25-8a66-dbe43dd2eece","JobId":"counter-kjwgl","error":"Job counter-kjwgl does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:12.470Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"6efcf9f7-3559-4c25-8a66-dbe43dd2eece"}
{"level":"error","ts":"2026-07-13T15:59:12.470Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"6efcf9f7-3559-4c25-8a66-dbe43dd2eece","error":"Job counter-kjwgl does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:12.630Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"3b26600d-71bf-4206-9e86-34c2478f1b34","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:12.630Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"3b26600d-71bf-4206-9e86-34c2478f1b34","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:12.630Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"3b26600d-71bf-4206-9e86-34c2478f1b34","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:12.630Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"3b26600d-71bf-4206-9e86-34c2478f1b34"}
{"level":"error","ts":"2026-07-13T15:59:12.636Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"3b26600d-71bf-4206-9e86-34c2478f1b34","JobId":"counter-kjwgl","error":"Job counter-kjwgl does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:12.636Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"3b26600d-71bf-4206-9e86-34c2478f1b34"}
{"level":"error","ts":"2026-07-13T15:59:12.636Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"3b26600d-71bf-4206-9e86-34c2478f1b34","error":"Job counter-kjwgl does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:12.956Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"c24314fd-61b2-4c04-8d06-5c3397dfd367","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:12.956Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"c24314fd-61b2-4c04-8d06-5c3397dfd367","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:12.956Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"c24314fd-61b2-4c04-8d06-5c3397dfd367","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:12.957Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"c24314fd-61b2-4c04-8d06-5c3397dfd367"}
{"level":"error","ts":"2026-07-13T15:59:12.962Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"c24314fd-61b2-4c04-8d06-5c3397dfd367","JobId":"counter-kjwgl","error":"Job counter-kjwgl does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:12.962Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"c24314fd-61b2-4c04-8d06-5c3397dfd367"}
{"level":"error","ts":"2026-07-13T15:59:12.962Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"c24314fd-61b2-4c04-8d06-5c3397dfd367","error":"Job counter-kjwgl does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:13.276Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ddf4f182-6c33-401b-be7e-d46fc767264e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:13.276Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ddf4f182-6c33-401b-be7e-d46fc767264e","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:13.276Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ddf4f182-6c33-401b-be7e-d46fc767264e","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:13.276Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ddf4f182-6c33-401b-be7e-d46fc767264e"}
{"level":"error","ts":"2026-07-13T15:59:13.281Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ddf4f182-6c33-401b-be7e-d46fc767264e","JobId":"counter-kjwgl","error":"Job counter-kjwgl does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:13.281Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ddf4f182-6c33-401b-be7e-d46fc767264e"}
{"level":"error","ts":"2026-07-13T15:59:13.281Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ddf4f182-6c33-401b-be7e-d46fc767264e","error":"Job counter-kjwgl does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:13.603Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"568b4ee9-b7ac-41e5-99b9-5f7e8599908b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:13.603Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"568b4ee9-b7ac-41e5-99b9-5f7e8599908b","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:13.603Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"568b4ee9-b7ac-41e5-99b9-5f7e8599908b","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:13.604Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"568b4ee9-b7ac-41e5-99b9-5f7e8599908b"}
{"level":"error","ts":"2026-07-13T15:59:13.608Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"568b4ee9-b7ac-41e5-99b9-5f7e8599908b","JobId":"counter-kjwgl","error":"Job counter-kjwgl does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:13.609Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"568b4ee9-b7ac-41e5-99b9-5f7e8599908b"}
{"level":"error","ts":"2026-07-13T15:59:13.609Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"568b4ee9-b7ac-41e5-99b9-5f7e8599908b","error":"Job counter-kjwgl does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:14.240Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"22ff2fbe-c92c-4260-953e-7d2382af4338"}
{"level":"info","ts":"2026-07-13T15:59:14.240Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"22ff2fbe-c92c-4260-953e-7d2382af4338"}
{"level":"info","ts":"2026-07-13T15:59:14.240Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"22ff2fbe-c92c-4260-953e-7d2382af4338","1 head service found":"counter-6m4r2-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:14.241Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"22ff2fbe-c92c-4260-953e-7d2382af4338","Found 1 head Pod":"counter-6m4r2-head-767d2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:14.241Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"22ff2fbe-c92c-4260-953e-7d2382af4338","head Pod":"counter-6m4r2-head-767d2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6m4r2-head-767d2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:14.241Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"22ff2fbe-c92c-4260-953e-7d2382af4338","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:14.241Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"22ff2fbe-c92c-4260-953e-7d2382af4338","worker Pod":"counter-6m4r2-small-group-worker-nlscq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-6m4r2-small-group-worker-nlscq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:14.241Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"22ff2fbe-c92c-4260-953e-7d2382af4338","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:14.241Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"22ff2fbe-c92c-4260-953e-7d2382af4338","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:14.241Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"22ff2fbe-c92c-4260-953e-7d2382af4338","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:14.241Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"22ff2fbe-c92c-4260-953e-7d2382af4338","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:59:14.241Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"22ff2fbe-c92c-4260-953e-7d2382af4338","seconds":300}
{"level":"info","ts":"2026-07-13T15:59:16.170Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"08cdaf9a-6177-4e46-8a6b-f469b66e4025","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:16.170Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"08cdaf9a-6177-4e46-8a6b-f469b66e4025","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:16.170Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"08cdaf9a-6177-4e46-8a6b-f469b66e4025","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:16.170Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"08cdaf9a-6177-4e46-8a6b-f469b66e4025"}
{"level":"error","ts":"2026-07-13T15:59:16.175Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"08cdaf9a-6177-4e46-8a6b-f469b66e4025","JobId":"counter-kjwgl","error":"Job counter-kjwgl does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:16.175Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"08cdaf9a-6177-4e46-8a6b-f469b66e4025"}
{"level":"error","ts":"2026-07-13T15:59:16.175Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"08cdaf9a-6177-4e46-8a6b-f469b66e4025","error":"Job counter-kjwgl does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:21.295Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"67f3b442-33aa-4353-b468-e36053166ceb","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:21.295Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"67f3b442-33aa-4353-b468-e36053166ceb","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:21.295Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"67f3b442-33aa-4353-b468-e36053166ceb","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:21.296Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"67f3b442-33aa-4353-b468-e36053166ceb"}
{"level":"info","ts":"2026-07-13T15:59:21.301Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"67f3b442-33aa-4353-b468-e36053166ceb","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","dashboardURL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:59:20Z"},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","dashboardURL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:12Z","stateTransitionTimes":{"ready":"2026-07-13T15:59:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:59:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:59:21.301Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"67f3b442-33aa-4353-b468-e36053166ceb","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T15:59:21.314Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"d9859d50-4118-4d9d-bcb3-95dc08e3e769","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:21.314Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"d9859d50-4118-4d9d-bcb3-95dc08e3e769","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:21.314Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"d9859d50-4118-4d9d-bcb3-95dc08e3e769","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:21.314Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"d9859d50-4118-4d9d-bcb3-95dc08e3e769"}
{"level":"info","ts":"2026-07-13T15:59:21.316Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"d9859d50-4118-4d9d-bcb3-95dc08e3e769","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:59:20Z"},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","dashboardURL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:12Z","stateTransitionTimes":{"ready":"2026-07-13T15:59:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:59:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:59:20Z"},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","dashboardURL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:12Z","stateTransitionTimes":{"ready":"2026-07-13T15:59:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:59:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:59:24.315Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1f9fb6ac-8f03-4035-810b-dabf606dee07","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:24.315Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1f9fb6ac-8f03-4035-810b-dabf606dee07","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:24.315Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1f9fb6ac-8f03-4035-810b-dabf606dee07","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:24.315Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1f9fb6ac-8f03-4035-810b-dabf606dee07"}
{"level":"info","ts":"2026-07-13T15:59:24.318Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1f9fb6ac-8f03-4035-810b-dabf606dee07","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:59:20Z"},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","dashboardURL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:12Z","stateTransitionTimes":{"ready":"2026-07-13T15:59:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:59:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:59:20Z","endTime":"2026-07-13T15:59:24Z"},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","dashboardURL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:12Z","stateTransitionTimes":{"ready":"2026-07-13T15:59:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:59:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:59:24.318Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1f9fb6ac-8f03-4035-810b-dabf606dee07","old JobStatus":"PENDING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T15:59:24.334Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f89b2fbc-1255-4c2b-a386-ebe85db71a11","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:24.334Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f89b2fbc-1255-4c2b-a386-ebe85db71a11","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:24.334Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f89b2fbc-1255-4c2b-a386-ebe85db71a11","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:24.334Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f89b2fbc-1255-4c2b-a386-ebe85db71a11"}
{"level":"info","ts":"2026-07-13T15:59:24.337Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"f89b2fbc-1255-4c2b-a386-ebe85db71a11","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:59:20Z","endTime":"2026-07-13T15:59:24Z"},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","dashboardURL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:12Z","stateTransitionTimes":{"ready":"2026-07-13T15:59:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:59:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:59:20Z","endTime":"2026-07-13T15:59:24Z"},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","dashboardURL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:12Z","stateTransitionTimes":{"ready":"2026-07-13T15:59:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:59:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:59:27.334Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"bb38441a-f739-4eae-8097-392b8711146f","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:27.334Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"bb38441a-f739-4eae-8097-392b8711146f","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:27.334Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"bb38441a-f739-4eae-8097-392b8711146f","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:27.334Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"bb38441a-f739-4eae-8097-392b8711146f"}
{"level":"info","ts":"2026-07-13T15:59:27.337Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"bb38441a-f739-4eae-8097-392b8711146f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:59:20Z","endTime":"2026-07-13T15:59:24Z"},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","dashboardURL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:12Z","stateTransitionTimes":{"ready":"2026-07-13T15:59:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:59:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:59:20Z","endTime":"2026-07-13T15:59:24Z"},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","dashboardURL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:12Z","stateTransitionTimes":{"ready":"2026-07-13T15:59:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:59:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:59:30.338Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ddc24e51-0dcd-4778-9a29-5eae2fc6cd8a","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:30.338Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ddc24e51-0dcd-4778-9a29-5eae2fc6cd8a","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:30.338Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ddc24e51-0dcd-4778-9a29-5eae2fc6cd8a","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:30.338Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ddc24e51-0dcd-4778-9a29-5eae2fc6cd8a"}
{"level":"info","ts":"2026-07-13T15:59:30.341Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"ddc24e51-0dcd-4778-9a29-5eae2fc6cd8a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:59:20Z","endTime":"2026-07-13T15:59:24Z"},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","dashboardURL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:12Z","stateTransitionTimes":{"ready":"2026-07-13T15:59:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:59:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:59:20Z","endTime":"2026-07-13T15:59:24Z"},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","dashboardURL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:12Z","stateTransitionTimes":{"ready":"2026-07-13T15:59:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:59:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:59:31.071Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"231b7949-7706-4c28-8f8f-bca5c775485d","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:31.071Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"231b7949-7706-4c28-8f8f-bca5c775485d","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:31.071Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"231b7949-7706-4c28-8f8f-bca5c775485d","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:31.072Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"231b7949-7706-4c28-8f8f-bca5c775485d"}
{"level":"info","ts":"2026-07-13T15:59:31.075Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"231b7949-7706-4c28-8f8f-bca5c775485d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:59:20Z","endTime":"2026-07-13T15:59:24Z"},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","dashboardURL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:12Z","stateTransitionTimes":{"ready":"2026-07-13T15:59:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:59:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:59:20Z","endTime":"2026-07-13T15:59:24Z"},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","dashboardURL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:12Z","stateTransitionTimes":{"ready":"2026-07-13T15:59:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:59:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:59:32.079Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"8f806c06-daac-45ee-9131-9ae0f5f8b60d","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:32.079Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"8f806c06-daac-45ee-9131-9ae0f5f8b60d","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:32.079Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"8f806c06-daac-45ee-9131-9ae0f5f8b60d","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:32.080Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"8f806c06-daac-45ee-9131-9ae0f5f8b60d"}
{"level":"info","ts":"2026-07-13T15:59:32.082Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"8f806c06-daac-45ee-9131-9ae0f5f8b60d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:59:20Z","endTime":"2026-07-13T15:59:24Z"},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","dashboardURL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:12Z","stateTransitionTimes":{"ready":"2026-07-13T15:59:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:59:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:59:20Z","endTime":"2026-07-13T15:59:24Z"},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","dashboardURL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:12Z","stateTransitionTimes":{"ready":"2026-07-13T15:59:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:59:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:59:32.096Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"d9bf2b5b-b4fd-431d-bb8d-6bdcaee0e379","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:32.096Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"d9bf2b5b-b4fd-431d-bb8d-6bdcaee0e379","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:32.096Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"d9bf2b5b-b4fd-431d-bb8d-6bdcaee0e379","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:32.096Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"d9bf2b5b-b4fd-431d-bb8d-6bdcaee0e379"}
{"level":"info","ts":"2026-07-13T15:59:32.098Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"d9bf2b5b-b4fd-431d-bb8d-6bdcaee0e379","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:59:20Z","endTime":"2026-07-13T15:59:24Z"},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","dashboardURL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:57:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:12Z","stateTransitionTimes":{"ready":"2026-07-13T15:59:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:59:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T15:59:20Z","endTime":"2026-07-13T15:59:24Z"},"jobId":"counter-kjwgl","rayClusterName":"counter-6m4r2","dashboardURL":"counter-6m4r2-head-svc.test-ns-88gnn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-07-13T15:57:20Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:12Z","stateTransitionTimes":{"ready":"2026-07-13T15:59:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"counter-6m4r2-head-767d2","serviceName":"counter-6m4r2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:58:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:59:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:57:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:57:20Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T15:59:32.099Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"d9bf2b5b-b4fd-431d-bb8d-6bdcaee0e379","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-07-13T15:59:32.111Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1ac9a32b-aec7-4e92-87ca-25c801b1ed7f","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:32.111Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1ac9a32b-aec7-4e92-87ca-25c801b1ed7f","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 15:59:32 +0000 UTC","Now":"2026-07-13T15:59:32.111Z","ShutdownTime":"2026-07-13T15:59:32.000Z"}
{"level":"info","ts":"2026-07-13T15:59:32.111Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1ac9a32b-aec7-4e92-87ca-25c801b1ed7f","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T15:59:32.119Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1ac9a32b-aec7-4e92-87ca-25c801b1ed7f","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:32.119Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1ac9a32b-aec7-4e92-87ca-25c801b1ed7f","isClusterDeleted":false}
{"level":"info","ts":"2026-07-13T15:59:32.119Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"1ac9a32b-aec7-4e92-87ca-25c801b1ed7f","RayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:32.120Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"8209716c-4959-4fed-957d-1d1c5b8e3874","namespacedName":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:32.120Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"8209716c-4959-4fed-957d-1d1c5b8e3874","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:32.120Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"7f2a342d-8eeb-42cd-8422-8d753423f315","rayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:32.120Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"55b007c6-9c84-45c3-9ec0-e4ee292e096d","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:32.120Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"dc9455e4-7fee-474d-8ee2-31131ad774cc"}
{"level":"info","ts":"2026-07-13T15:59:32.120Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"55b007c6-9c84-45c3-9ec0-e4ee292e096d","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 15:59:32 +0000 UTC","Now":"2026-07-13T15:59:32.120Z","ShutdownTime":"2026-07-13T15:59:32.000Z"}
{"level":"info","ts":"2026-07-13T15:59:32.120Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"55b007c6-9c84-45c3-9ec0-e4ee292e096d","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T15:59:32.120Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"reconcileID":"5f56546c-ab5b-4085-8434-618647d47029"}
{"level":"info","ts":"2026-07-13T15:59:32.120Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"55b007c6-9c84-45c3-9ec0-e4ee292e096d","RayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:32.120Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"55b007c6-9c84-45c3-9ec0-e4ee292e096d","isClusterDeleted":false}
{"level":"info","ts":"2026-07-13T15:59:32.120Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"55b007c6-9c84-45c3-9ec0-e4ee292e096d","RayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:32.120Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"8209716c-4959-4fed-957d-1d1c5b8e3874","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:32.120Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"7f2a342d-8eeb-42cd-8422-8d753423f315","pod":"counter-6m4r2-head-767d2","ip":"10.133.0.26"}
{"level":"info","ts":"2026-07-13T15:59:32.120Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"7f2a342d-8eeb-42cd-8422-8d753423f315","pod":"counter-6m4r2-small-group-worker-nlscq","ip":"10.134.0.30"}
{"level":"info","ts":"2026-07-13T15:59:32.120Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"8209716c-4959-4fed-957d-1d1c5b8e3874","namespace":"test-ns-88gnn","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-13T15:59:32.120Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"7f2a342d-8eeb-42cd-8422-8d753423f315","rayCluster":"counter-6m4r2","podIPs":["10.133.0.26","10.134.0.30"]}
{"level":"info","ts":"2026-07-13T15:59:32.128Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"8209716c-4959-4fed-957d-1d1c5b8e3874","name":"kuberay-gateway-access","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:59:32.133Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"8209716c-4959-4fed-957d-1d1c5b8e3874","name":"test-ns-88gnn-counter-6m4r2","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-13T15:59:32.137Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"8209716c-4959-4fed-957d-1d1c5b8e3874","configMap":"kube-rbac-proxy-config-counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:32.141Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"8209716c-4959-4fed-957d-1d1c5b8e3874","serviceAccount":"counter-6m4r2-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:59:32.161Z","logger":"raycluster-resource","msg":"default","name":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:32.161Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-6m4r2","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:59:32.171Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"8209716c-4959-4fed-957d-1d1c5b8e3874","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:32.171Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"26e2b870-0ad8-4c8a-b101-6f89cbd082ec","namespacedName":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:32.171Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"26e2b870-0ad8-4c8a-b101-6f89cbd082ec","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:32.171Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"26e2b870-0ad8-4c8a-b101-6f89cbd082ec","cluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:32.171Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"26e2b870-0ad8-4c8a-b101-6f89cbd082ec","namespace":"test-ns-88gnn","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-13T15:59:32.171Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"26e2b870-0ad8-4c8a-b101-6f89cbd082ec","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-13T15:59:32.171Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"00379e3c-d7aa-4d29-87b0-9b3df0a99e3c","namespace":"test-ns-88gnn","clusterName":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:32.171Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"828a9bed-7944-41c6-ad1d-0218431eb961","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:32.171Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"828a9bed-7944-41c6-ad1d-0218431eb961","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 15:59:32 +0000 UTC","Now":"2026-07-13T15:59:32.171Z","ShutdownTime":"2026-07-13T15:59:32.000Z"}
{"level":"info","ts":"2026-07-13T15:59:32.171Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"828a9bed-7944-41c6-ad1d-0218431eb961","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T15:59:32.171Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"828a9bed-7944-41c6-ad1d-0218431eb961","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:32.171Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"828a9bed-7944-41c6-ad1d-0218431eb961","isClusterDeleted":true}
{"level":"info","ts":"2026-07-13T15:59:32.172Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"828a9bed-7944-41c6-ad1d-0218431eb961","RayCluster":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:32.171Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"8b5d10f9-bfd0-462d-883b-81883e6194a0"}
{"level":"error","ts":"2026-07-13T15:59:32.222Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"26e2b870-0ad8-4c8a-b101-6f89cbd082ec","error":"rayclusters.ray.io \"counter-6m4r2\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:32.222Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"26e2b870-0ad8-4c8a-b101-6f89cbd082ec"}
{"level":"error","ts":"2026-07-13T15:59:32.222Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"26e2b870-0ad8-4c8a-b101-6f89cbd082ec","error":"rayclusters.ray.io \"counter-6m4r2\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:32.222Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"6dcc3d90-ad7b-4b00-b4da-9e309636140b","namespacedName":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:32.222Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"6dcc3d90-ad7b-4b00-b4da-9e309636140b"}
{"level":"info","ts":"2026-07-13T15:59:32.222Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"6dcc3d90-ad7b-4b00-b4da-9e309636140b","namespace":"test-ns-88gnn","authClusters":0}
{"level":"info","ts":"2026-07-13T15:59:32.222Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"6dcc3d90-ad7b-4b00-b4da-9e309636140b","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:59:32.222Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"58ae41e7-37c3-42cd-be92-38c30339c814","namespacedName":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:32.222Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"58ae41e7-37c3-42cd-be92-38c30339c814"}
{"level":"info","ts":"2026-07-13T15:59:32.222Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"58ae41e7-37c3-42cd-be92-38c30339c814","namespace":"test-ns-88gnn","authClusters":0}
{"level":"info","ts":"2026-07-13T15:59:32.222Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"58ae41e7-37c3-42cd-be92-38c30339c814","namespace":"test-ns-88gnn"}
{"level":"info","ts":"2026-07-13T15:59:32.222Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"6f6fa566-856d-4780-b53a-ba9dea04a33d"}
{"level":"info","ts":"2026-07-13T15:59:32.222Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"70bb27f2-0248-42f4-a3e2-c2b56fdb38b3"}
{"level":"info","ts":"2026-07-13T15:59:32.321Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"f2e9cc5c-aa00-4de9-962c-4a5f3eff34e2","namespacedName":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"}}
{"level":"info","ts":"2026-07-13T15:59:32.321Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"f2e9cc5c-aa00-4de9-962c-4a5f3eff34e2"}
{"level":"info","ts":"2026-07-13T15:59:32.321Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"f2e9cc5c-aa00-4de9-962c-4a5f3eff34e2","namespace":"test-ns-88gnn","authClusters":0}
{"level":"info","ts":"2026-07-13T15:59:32.321Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"f2e9cc5c-aa00-4de9-962c-4a5f3eff34e2","namespace":"test-ns-88gnn"}
{"level":"error","ts":"2026-07-13T15:59:32.322Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"00379e3c-d7aa-4d29-87b0-9b3df0a99e3c","issuer":"ray-selfsigned-issuer-counter-6m4r2","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-counter-6m4r2\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:59:32.422Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"00379e3c-d7aa-4d29-87b0-9b3df0a99e3c","certificate":"ray-head-cert-counter-6m4r2","error":"certificates.cert-manager.io \"ray-head-cert-counter-6m4r2\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:32.441Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"00379e3c-d7aa-4d29-87b0-9b3df0a99e3c","errorCount":2}
{"level":"info","ts":"2026-07-13T15:59:32.441Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"00379e3c-d7aa-4d29-87b0-9b3df0a99e3c"}
{"level":"error","ts":"2026-07-13T15:59:32.441Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"00379e3c-d7aa-4d29-87b0-9b3df0a99e3c","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-counter-6m4r2\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:32.446Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"476568eb-f2d3-40c6-a982-31e28310b921","namespace":"test-ns-88gnn","clusterName":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:32.446Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6m4r2","namespace":"test-ns-88gnn"},"namespace":"test-ns-88gnn","name":"counter-6m4r2","reconcileID":"476568eb-f2d3-40c6-a982-31e28310b921","namespace":"test-ns-88gnn","clusterName":"counter-6m4r2"}
{"level":"info","ts":"2026-07-13T15:59:33.051Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"e2a910da-b6b0-469b-a16d-f45011ff37b7","DeletionTimestamp":"2026-07-13 15:59:33 +0000 UTC"}
{"level":"info","ts":"2026-07-13T15:59:33.051Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"e2a910da-b6b0-469b-a16d-f45011ff37b7","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T15:59:33.067Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"2a1260be-b800-4553-aa92-9af94ee72352"}
{"level":"info","ts":"2026-07-13T15:59:33.071Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"336e50dc-95f6-4b51-b538-085ca4d70ff0"}
{"level":"info","ts":"2026-07-13T15:59:33.342Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-88gnn"},"reconcileID":"12c0e7ee-9407-4fcc-a45f-542c8cb7251f"}
{"level":"info","ts":"2026-07-13T15:59:34.942Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"fdd69def-a206-4e03-ba83-d481528e3709","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T15:59:34.942Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"fdd69def-a206-4e03-ba83-d481528e3709","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T15:59:34.942Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"fdd69def-a206-4e03-ba83-d481528e3709","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T15:59:34.942Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"fdd69def-a206-4e03-ba83-d481528e3709","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T15:59:35.133Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"87a319c1-bec2-4df7-b1b6-be4861b0a701","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:35.133Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"87a319c1-bec2-4df7-b1b6-be4861b0a701","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T15:59:35.146Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"87a319c1-bec2-4df7-b1b6-be4861b0a701"}
{"level":"info","ts":"2026-07-13T15:59:35.146Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"87a319c1-bec2-4df7-b1b6-be4861b0a701","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-13T15:59:35.146Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"87a319c1-bec2-4df7-b1b6-be4861b0a701","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:59:35.147Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"87a319c1-bec2-4df7-b1b6-be4861b0a701","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T15:59:35.160Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"ef1061ea-7862-47e7-8592-4bb9ebd405b4","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:35.160Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"ef1061ea-7862-47e7-8592-4bb9ebd405b4"}
{"level":"info","ts":"2026-07-13T15:59:35.160Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"ef1061ea-7862-47e7-8592-4bb9ebd405b4","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-13T15:59:35.160Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"ef1061ea-7862-47e7-8592-4bb9ebd405b4","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-tr7d8","rayClusterName":"counter-bwd98","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T15:59:35.160Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"ef1061ea-7862-47e7-8592-4bb9ebd405b4","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T15:59:35.167Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"ef1061ea-7862-47e7-8592-4bb9ebd405b4","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-13T15:59:35.167Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"ef1061ea-7862-47e7-8592-4bb9ebd405b4"}
{"level":"error","ts":"2026-07-13T15:59:35.167Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"ef1061ea-7862-47e7-8592-4bb9ebd405b4","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:35.167Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e7c9fcd2-8bc0-4a93-a3b7-836d62dd492c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:35.167Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e7c9fcd2-8bc0-4a93-a3b7-836d62dd492c","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.167Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e7c9fcd2-8bc0-4a93-a3b7-836d62dd492c","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:35.167Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e7c9fcd2-8bc0-4a93-a3b7-836d62dd492c","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:35.171Z","logger":"raycluster-resource","msg":"default","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.171Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-rn54g","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T15:59:35.180Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e7c9fcd2-8bc0-4a93-a3b7-836d62dd492c","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:35.180Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e7c9fcd2-8bc0-4a93-a3b7-836d62dd492c"}
{"level":"info","ts":"2026-07-13T15:59:35.180Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e7c9fcd2-8bc0-4a93-a3b7-836d62dd492c","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T15:59:35.181Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"fa2cb2ec-460e-45aa-98f1-81401a193710","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:35.181Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"fa2cb2ec-460e-45aa-98f1-81401a193710","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.181Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"fa2cb2ec-460e-45aa-98f1-81401a193710","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:35.181Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"fa2cb2ec-460e-45aa-98f1-81401a193710","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:35.181Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"e580eee3-a179-45e3-b071-fac12628b227","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.181Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"33a35a4d-191a-4993-a9fe-fd70da778c4e"}
{"level":"info","ts":"2026-07-13T15:59:35.181Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"33a35a4d-191a-4993-a9fe-fd70da778c4e"}
{"level":"info","ts":"2026-07-13T15:59:35.181Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"d02b3971-9089-4d3e-812f-736793523393","namespacedName":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:35.181Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"8723b47a-38d4-4395-bcdb-72ac25a617d4","rayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.181Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"8723b47a-38d4-4395-bcdb-72ac25a617d4","rayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.181Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"d02b3971-9089-4d3e-812f-736793523393","mode":"IntegratedOAuth","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.181Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"d02b3971-9089-4d3e-812f-736793523393","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.181Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"d02b3971-9089-4d3e-812f-736793523393","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.221Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"8723b47a-38d4-4395-bcdb-72ac25a617d4","rayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.221Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"8723b47a-38d4-4395-bcdb-72ac25a617d4","rayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.221Z","logger":"raycluster-resource","msg":"default","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.221Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-rn54g","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T15:59:35.225Z","logger":"raycluster-resource","msg":"default","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.225Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-rn54g","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T15:59:35.226Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"33a35a4d-191a-4993-a9fe-fd70da778c4e","name":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:35.226Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"33a35a4d-191a-4993-a9fe-fd70da778c4e","error":"head certificate secret ray-head-secret-counter-rn54g not found"}
{"level":"error","ts":"2026-07-13T15:59:35.226Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"33a35a4d-191a-4993-a9fe-fd70da778c4e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:35.226Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"33a35a4d-191a-4993-a9fe-fd70da778c4e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found"}
{"level":"info","ts":"2026-07-13T15:59:35.226Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"33a35a4d-191a-4993-a9fe-fd70da778c4e","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:35.226Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"33a35a4d-191a-4993-a9fe-fd70da778c4e","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:35.226Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"33a35a4d-191a-4993-a9fe-fd70da778c4e","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:35.226Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"33a35a4d-191a-4993-a9fe-fd70da778c4e","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:59:35.227Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"33a35a4d-191a-4993-a9fe-fd70da778c4e","name":"counter-rn54g","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:59:35.227Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"e580eee3-a179-45e3-b071-fac12628b227","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.227Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"959ff14b-7301-48d1-b417-4f8bbd529386","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.234Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"8723b47a-38d4-4395-bcdb-72ac25a617d4","rayCluster":"counter-rn54g","secret":"ca-secret"}
{"level":"info","ts":"2026-07-13T15:59:35.235Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"8723b47a-38d4-4395-bcdb-72ac25a617d4","rayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.238Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"fa2cb2ec-460e-45aa-98f1-81401a193710"}
{"level":"error","ts":"2026-07-13T15:59:35.238Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"fa2cb2ec-460e-45aa-98f1-81401a193710","error":"rayclusters.ray.io \"counter-rn54g\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:35.238Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"fedceb8d-00e9-4a6a-bf38-2a5846329e55","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:35.238Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"fedceb8d-00e9-4a6a-bf38-2a5846329e55","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.238Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"fedceb8d-00e9-4a6a-bf38-2a5846329e55","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:35.238Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"fedceb8d-00e9-4a6a-bf38-2a5846329e55"}
{"level":"info","ts":"2026-07-13T15:59:35.238Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"fedceb8d-00e9-4a6a-bf38-2a5846329e55","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T15:59:35.239Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"33a35a4d-191a-4993-a9fe-fd70da778c4e","name":"counter-rn54g","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-rn54g\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-x5qvg","name":"counter-rn54g"}}
{"level":"info","ts":"2026-07-13T15:59:35.239Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"33a35a4d-191a-4993-a9fe-fd70da778c4e"}
{"level":"error","ts":"2026-07-13T15:59:35.239Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"33a35a4d-191a-4993-a9fe-fd70da778c4e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:35.240Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"956219c2-218b-47a0-99d3-3f76152d59c1"}
{"level":"info","ts":"2026-07-13T15:59:35.240Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"956219c2-218b-47a0-99d3-3f76152d59c1"}
{"level":"info","ts":"2026-07-13T15:59:35.240Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"956219c2-218b-47a0-99d3-3f76152d59c1","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:35.240Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"956219c2-218b-47a0-99d3-3f76152d59c1","error":"head certificate secret ray-head-secret-counter-rn54g not found"}
{"level":"error","ts":"2026-07-13T15:59:35.240Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"956219c2-218b-47a0-99d3-3f76152d59c1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:35.240Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"956219c2-218b-47a0-99d3-3f76152d59c1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found"}
{"level":"info","ts":"2026-07-13T15:59:35.240Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"956219c2-218b-47a0-99d3-3f76152d59c1","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:35.240Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"956219c2-218b-47a0-99d3-3f76152d59c1","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:35.240Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"956219c2-218b-47a0-99d3-3f76152d59c1","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:35.240Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"956219c2-218b-47a0-99d3-3f76152d59c1","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:59:35.240Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"956219c2-218b-47a0-99d3-3f76152d59c1","name":"counter-rn54g","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:59:35.321Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"7d376dcc-7378-486d-9af8-b3473908f092","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:35.321Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"7d376dcc-7378-486d-9af8-b3473908f092","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.321Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"7d376dcc-7378-486d-9af8-b3473908f092","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:35.321Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"d02b3971-9089-4d3e-812f-736793523393","name":"counter-rn54g-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:59:35.321Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"8723b47a-38d4-4395-bcdb-72ac25a617d4","rayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.321Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"7d376dcc-7378-486d-9af8-b3473908f092"}
{"level":"info","ts":"2026-07-13T15:59:35.321Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"7d376dcc-7378-486d-9af8-b3473908f092","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T15:59:35.321Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"959ff14b-7301-48d1-b417-4f8bbd529386","name":"counter-rn54g-head"}
{"level":"info","ts":"2026-07-13T15:59:35.326Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"d02b3971-9089-4d3e-812f-736793523393","name":"kuberay-gateway-access","namespace":"test-ns-x5qvg","operation":"created"}
{"level":"info","ts":"2026-07-13T15:59:35.333Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"8723b47a-38d4-4395-bcdb-72ac25a617d4","rayCluster":"counter-rn54g","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:59:35.333Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"956219c2-218b-47a0-99d3-3f76152d59c1"}
{"level":"error","ts":"2026-07-13T15:59:35.333Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"956219c2-218b-47a0-99d3-3f76152d59c1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:35.333Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5afe55ae-49bd-4c75-b82b-fc3f3c6074dc"}
{"level":"info","ts":"2026-07-13T15:59:35.333Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5afe55ae-49bd-4c75-b82b-fc3f3c6074dc"}
{"level":"info","ts":"2026-07-13T15:59:35.333Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5afe55ae-49bd-4c75-b82b-fc3f3c6074dc","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:35.333Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5afe55ae-49bd-4c75-b82b-fc3f3c6074dc","error":"head certificate secret ray-head-secret-counter-rn54g not found"}
{"level":"error","ts":"2026-07-13T15:59:35.333Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5afe55ae-49bd-4c75-b82b-fc3f3c6074dc","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:35.333Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5afe55ae-49bd-4c75-b82b-fc3f3c6074dc","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found"}
{"level":"info","ts":"2026-07-13T15:59:35.333Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5afe55ae-49bd-4c75-b82b-fc3f3c6074dc","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:35.333Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5afe55ae-49bd-4c75-b82b-fc3f3c6074dc","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:35.333Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5afe55ae-49bd-4c75-b82b-fc3f3c6074dc","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:35.333Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5afe55ae-49bd-4c75-b82b-fc3f3c6074dc","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:59:35.333Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5afe55ae-49bd-4c75-b82b-fc3f3c6074dc","name":"counter-rn54g","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:59:35.333Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"959ff14b-7301-48d1-b417-4f8bbd529386","name":"counter-rn54g-workers"}
{"level":"info","ts":"2026-07-13T15:59:35.333Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"959ff14b-7301-48d1-b417-4f8bbd529386","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.333Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"e8b60dbf-a99b-4746-86d5-8753cd695043","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.334Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"780fa5d7-2d8f-4aa8-a3fb-11b130f56fdf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:35.334Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"780fa5d7-2d8f-4aa8-a3fb-11b130f56fdf","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.334Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"780fa5d7-2d8f-4aa8-a3fb-11b130f56fdf","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:35.334Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"780fa5d7-2d8f-4aa8-a3fb-11b130f56fdf"}
{"level":"info","ts":"2026-07-13T15:59:35.334Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"780fa5d7-2d8f-4aa8-a3fb-11b130f56fdf","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T15:59:35.421Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5afe55ae-49bd-4c75-b82b-fc3f3c6074dc","name":"counter-rn54g","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-rn54g\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-x5qvg","name":"counter-rn54g"}}
{"level":"info","ts":"2026-07-13T15:59:35.421Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5afe55ae-49bd-4c75-b82b-fc3f3c6074dc"}
{"level":"info","ts":"2026-07-13T15:59:35.421Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"d02b3971-9089-4d3e-812f-736793523393","name":"test-ns-x5qvg-counter-rn54g","operation":"created"}
{"level":"error","ts":"2026-07-13T15:59:35.421Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5afe55ae-49bd-4c75-b82b-fc3f3c6074dc","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:35.421Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"cbfae6f5-eae8-4954-9912-78b2de202371"}
{"level":"info","ts":"2026-07-13T15:59:35.421Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"cbfae6f5-eae8-4954-9912-78b2de202371"}
{"level":"info","ts":"2026-07-13T15:59:35.421Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"8723b47a-38d4-4395-bcdb-72ac25a617d4","rayCluster":"counter-rn54g","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:59:35.421Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"cbfae6f5-eae8-4954-9912-78b2de202371","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:35.421Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"cbfae6f5-eae8-4954-9912-78b2de202371","error":"head certificate secret ray-head-secret-counter-rn54g not found"}
{"level":"error","ts":"2026-07-13T15:59:35.421Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"cbfae6f5-eae8-4954-9912-78b2de202371","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:35.421Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"cbfae6f5-eae8-4954-9912-78b2de202371","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found"}
{"level":"info","ts":"2026-07-13T15:59:35.421Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"8723b47a-38d4-4395-bcdb-72ac25a617d4"}
{"level":"info","ts":"2026-07-13T15:59:35.421Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"cbfae6f5-eae8-4954-9912-78b2de202371","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:35.422Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"cbfae6f5-eae8-4954-9912-78b2de202371","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:35.422Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"fdfb4be1-ff36-41d8-8027-cd70a2cb27da","rayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.422Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"cbfae6f5-eae8-4954-9912-78b2de202371","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:35.422Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"cbfae6f5-eae8-4954-9912-78b2de202371"}
{"level":"error","ts":"2026-07-13T15:59:35.422Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"cbfae6f5-eae8-4954-9912-78b2de202371","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:35.422Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"fdfb4be1-ff36-41d8-8027-cd70a2cb27da"}
{"level":"info","ts":"2026-07-13T15:59:35.427Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"d02b3971-9089-4d3e-812f-736793523393","name":"kube-rbac-proxy-config-counter-rn54g","operation":"created"}
{"level":"info","ts":"2026-07-13T15:59:35.427Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"e8b60dbf-a99b-4746-86d5-8753cd695043","name":"counter-rn54g-head"}
{"level":"error","ts":"2026-07-13T15:59:35.436Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"d02b3971-9089-4d3e-812f-736793523393","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-rn54g\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:35.437Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"d02b3971-9089-4d3e-812f-736793523393"}
{"level":"error","ts":"2026-07-13T15:59:35.437Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"d02b3971-9089-4d3e-812f-736793523393","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-rn54g\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:35.437Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"4ab88efe-f50b-41aa-be06-8f041c912a24","namespacedName":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:35.437Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"4ab88efe-f50b-41aa-be06-8f041c912a24","mode":"IntegratedOAuth","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.437Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"4ab88efe-f50b-41aa-be06-8f041c912a24","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.441Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"e8b60dbf-a99b-4746-86d5-8753cd695043","name":"counter-rn54g-workers"}
{"level":"info","ts":"2026-07-13T15:59:35.441Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"e8b60dbf-a99b-4746-86d5-8753cd695043","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.441Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"822eb3b0-ad7a-47a7-a42a-98afbbae44d4","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.441Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"eebb7b76-b5db-416a-b880-880809427213"}
{"level":"info","ts":"2026-07-13T15:59:35.441Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"eebb7b76-b5db-416a-b880-880809427213"}
{"level":"info","ts":"2026-07-13T15:59:35.441Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"eebb7b76-b5db-416a-b880-880809427213","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:35.441Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"eebb7b76-b5db-416a-b880-880809427213","error":"head certificate secret ray-head-secret-counter-rn54g not found"}
{"level":"error","ts":"2026-07-13T15:59:35.441Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"eebb7b76-b5db-416a-b880-880809427213","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:35.441Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"eebb7b76-b5db-416a-b880-880809427213","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found"}
{"level":"info","ts":"2026-07-13T15:59:35.441Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"eebb7b76-b5db-416a-b880-880809427213","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:35.442Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"eebb7b76-b5db-416a-b880-880809427213","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:35.442Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"eebb7b76-b5db-416a-b880-880809427213","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:35.442Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"eebb7b76-b5db-416a-b880-880809427213"}
{"level":"error","ts":"2026-07-13T15:59:35.442Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"eebb7b76-b5db-416a-b880-880809427213","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:35.448Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"4ab88efe-f50b-41aa-be06-8f041c912a24","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.448Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"115c54f6-99f5-42e6-8a17-7e422840daef","namespacedName":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:35.448Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"115c54f6-99f5-42e6-8a17-7e422840daef","mode":"IntegratedOAuth","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.448Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"115c54f6-99f5-42e6-8a17-7e422840daef","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.448Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"522ec45f-438b-4f0c-ba75-59a9465f9a93","rayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.448Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"c69d28cd-1c67-435d-b790-d5dfba69c56f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:35.448Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"c69d28cd-1c67-435d-b790-d5dfba69c56f","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.448Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"c69d28cd-1c67-435d-b790-d5dfba69c56f","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:35.448Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"522ec45f-438b-4f0c-ba75-59a9465f9a93"}
{"level":"info","ts":"2026-07-13T15:59:35.448Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"c69d28cd-1c67-435d-b790-d5dfba69c56f"}
{"level":"info","ts":"2026-07-13T15:59:35.448Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"c69d28cd-1c67-435d-b790-d5dfba69c56f","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T15:59:35.452Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"822eb3b0-ad7a-47a7-a42a-98afbbae44d4","name":"counter-rn54g-head"}
{"level":"error","ts":"2026-07-13T15:59:35.521Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"115c54f6-99f5-42e6-8a17-7e422840daef","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-rn54g\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:35.521Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"115c54f6-99f5-42e6-8a17-7e422840daef"}
{"level":"error","ts":"2026-07-13T15:59:35.521Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"115c54f6-99f5-42e6-8a17-7e422840daef","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-rn54g\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:35.522Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3db84457-f1a7-4027-a764-cf496e66dc3b"}
{"level":"info","ts":"2026-07-13T15:59:35.522Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3db84457-f1a7-4027-a764-cf496e66dc3b"}
{"level":"info","ts":"2026-07-13T15:59:35.522Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3db84457-f1a7-4027-a764-cf496e66dc3b","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:35.522Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3db84457-f1a7-4027-a764-cf496e66dc3b","error":"head certificate secret ray-head-secret-counter-rn54g not found"}
{"level":"error","ts":"2026-07-13T15:59:35.522Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3db84457-f1a7-4027-a764-cf496e66dc3b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:35.522Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3db84457-f1a7-4027-a764-cf496e66dc3b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found"}
{"level":"info","ts":"2026-07-13T15:59:35.522Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3db84457-f1a7-4027-a764-cf496e66dc3b","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:35.522Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3db84457-f1a7-4027-a764-cf496e66dc3b","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:35.522Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3db84457-f1a7-4027-a764-cf496e66dc3b","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:35.522Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3db84457-f1a7-4027-a764-cf496e66dc3b"}
{"level":"error","ts":"2026-07-13T15:59:35.522Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3db84457-f1a7-4027-a764-cf496e66dc3b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:35.525Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"822eb3b0-ad7a-47a7-a42a-98afbbae44d4","name":"counter-rn54g-workers"}
{"level":"info","ts":"2026-07-13T15:59:35.525Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"822eb3b0-ad7a-47a7-a42a-98afbbae44d4","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.525Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"da4eaaae-3447-44dc-b9fa-ab30b7860f8d","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.527Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"112a944a-2e99-49f2-abae-3a8be3590520","namespacedName":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:35.527Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"112a944a-2e99-49f2-abae-3a8be3590520","mode":"IntegratedOAuth","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.527Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"112a944a-2e99-49f2-abae-3a8be3590520","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.534Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"da4eaaae-3447-44dc-b9fa-ab30b7860f8d","name":"counter-rn54g-head"}
{"level":"info","ts":"2026-07-13T15:59:35.536Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"112a944a-2e99-49f2-abae-3a8be3590520","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.544Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"da4eaaae-3447-44dc-b9fa-ab30b7860f8d","name":"counter-rn54g-workers"}
{"level":"info","ts":"2026-07-13T15:59:35.545Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"da4eaaae-3447-44dc-b9fa-ab30b7860f8d","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:35.683Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c58c6728-13b2-4733-b079-9e88d3b4c019"}
{"level":"info","ts":"2026-07-13T15:59:35.683Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c58c6728-13b2-4733-b079-9e88d3b4c019"}
{"level":"info","ts":"2026-07-13T15:59:35.683Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c58c6728-13b2-4733-b079-9e88d3b4c019","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:35.683Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c58c6728-13b2-4733-b079-9e88d3b4c019","error":"head certificate secret ray-head-secret-counter-rn54g not found"}
{"level":"error","ts":"2026-07-13T15:59:35.683Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c58c6728-13b2-4733-b079-9e88d3b4c019","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:35.683Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c58c6728-13b2-4733-b079-9e88d3b4c019","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found"}
{"level":"info","ts":"2026-07-13T15:59:35.683Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c58c6728-13b2-4733-b079-9e88d3b4c019","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:35.684Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c58c6728-13b2-4733-b079-9e88d3b4c019","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:35.684Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c58c6728-13b2-4733-b079-9e88d3b4c019","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:35.684Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c58c6728-13b2-4733-b079-9e88d3b4c019"}
{"level":"error","ts":"2026-07-13T15:59:35.684Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c58c6728-13b2-4733-b079-9e88d3b4c019","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:36.004Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2d62238e-f639-4b52-ac79-1f831966c01f"}
{"level":"info","ts":"2026-07-13T15:59:36.004Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2d62238e-f639-4b52-ac79-1f831966c01f"}
{"level":"info","ts":"2026-07-13T15:59:36.004Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2d62238e-f639-4b52-ac79-1f831966c01f","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:36.004Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2d62238e-f639-4b52-ac79-1f831966c01f","error":"head certificate secret ray-head-secret-counter-rn54g not found"}
{"level":"error","ts":"2026-07-13T15:59:36.004Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2d62238e-f639-4b52-ac79-1f831966c01f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:36.004Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2d62238e-f639-4b52-ac79-1f831966c01f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found"}
{"level":"info","ts":"2026-07-13T15:59:36.004Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2d62238e-f639-4b52-ac79-1f831966c01f","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:36.005Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2d62238e-f639-4b52-ac79-1f831966c01f","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:36.005Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2d62238e-f639-4b52-ac79-1f831966c01f","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:36.005Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2d62238e-f639-4b52-ac79-1f831966c01f"}
{"level":"error","ts":"2026-07-13T15:59:36.005Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2d62238e-f639-4b52-ac79-1f831966c01f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:36.645Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53cc42b8-0d09-4e44-8603-7edbd338cb14"}
{"level":"info","ts":"2026-07-13T15:59:36.645Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53cc42b8-0d09-4e44-8603-7edbd338cb14"}
{"level":"info","ts":"2026-07-13T15:59:36.645Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53cc42b8-0d09-4e44-8603-7edbd338cb14","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:36.645Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53cc42b8-0d09-4e44-8603-7edbd338cb14","error":"head certificate secret ray-head-secret-counter-rn54g not found"}
{"level":"error","ts":"2026-07-13T15:59:36.645Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53cc42b8-0d09-4e44-8603-7edbd338cb14","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:36.645Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53cc42b8-0d09-4e44-8603-7edbd338cb14","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found"}
{"level":"info","ts":"2026-07-13T15:59:36.645Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53cc42b8-0d09-4e44-8603-7edbd338cb14","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:36.646Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53cc42b8-0d09-4e44-8603-7edbd338cb14","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:36.646Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53cc42b8-0d09-4e44-8603-7edbd338cb14","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:36.646Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53cc42b8-0d09-4e44-8603-7edbd338cb14"}
{"level":"error","ts":"2026-07-13T15:59:36.646Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53cc42b8-0d09-4e44-8603-7edbd338cb14","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:37.794Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"58103ee6-6aa5-4933-b473-48cbf8f6e233"}
{"level":"info","ts":"2026-07-13T15:59:37.794Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"58103ee6-6aa5-4933-b473-48cbf8f6e233"}
{"level":"info","ts":"2026-07-13T15:59:37.794Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"58103ee6-6aa5-4933-b473-48cbf8f6e233","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:37.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"58103ee6-6aa5-4933-b473-48cbf8f6e233","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:37.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"58103ee6-6aa5-4933-b473-48cbf8f6e233","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:37.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"58103ee6-6aa5-4933-b473-48cbf8f6e233","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:37.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"58103ee6-6aa5-4933-b473-48cbf8f6e233","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:37.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"58103ee6-6aa5-4933-b473-48cbf8f6e233","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:37.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"58103ee6-6aa5-4933-b473-48cbf8f6e233","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:37.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"58103ee6-6aa5-4933-b473-48cbf8f6e233","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:37.794Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"58103ee6-6aa5-4933-b473-48cbf8f6e233","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:59:37.794Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"58103ee6-6aa5-4933-b473-48cbf8f6e233","seconds":300}
{"level":"info","ts":"2026-07-13T15:59:37.926Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"972a9113-af1a-458d-83d1-180ce86bbb7a"}
{"level":"info","ts":"2026-07-13T15:59:37.926Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"972a9113-af1a-458d-83d1-180ce86bbb7a"}
{"level":"info","ts":"2026-07-13T15:59:37.926Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"972a9113-af1a-458d-83d1-180ce86bbb7a","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:37.926Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"972a9113-af1a-458d-83d1-180ce86bbb7a","error":"head certificate secret ray-head-secret-counter-rn54g not found"}
{"level":"error","ts":"2026-07-13T15:59:37.926Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"972a9113-af1a-458d-83d1-180ce86bbb7a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:37.926Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"972a9113-af1a-458d-83d1-180ce86bbb7a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found"}
{"level":"info","ts":"2026-07-13T15:59:37.926Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"972a9113-af1a-458d-83d1-180ce86bbb7a","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:37.926Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"972a9113-af1a-458d-83d1-180ce86bbb7a","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:37.926Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"972a9113-af1a-458d-83d1-180ce86bbb7a","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"counter-rn54g","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T15:59:37.927Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"972a9113-af1a-458d-83d1-180ce86bbb7a"}
{"level":"error","ts":"2026-07-13T15:59:37.927Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"972a9113-af1a-458d-83d1-180ce86bbb7a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rn54g not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:38.322Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"4a61ae44-9e55-4e2c-99cd-013a9e80a506","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:38.322Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"4a61ae44-9e55-4e2c-99cd-013a9e80a506","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:38.322Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"4a61ae44-9e55-4e2c-99cd-013a9e80a506","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:38.322Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"4a61ae44-9e55-4e2c-99cd-013a9e80a506"}
{"level":"info","ts":"2026-07-13T15:59:38.322Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"4a61ae44-9e55-4e2c-99cd-013a9e80a506","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T15:59:40.487Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879"}
{"level":"info","ts":"2026-07-13T15:59:40.487Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879"}
{"level":"info","ts":"2026-07-13T15:59:40.487Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:40.487Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879"}
{"level":"info","ts":"2026-07-13T15:59:40.487Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:59:40.487Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879"}
{"level":"info","ts":"2026-07-13T15:59:40.487Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","mode":"IntegratedOAuth","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:40.487Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","cluster":"counter-rn54g","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T15:59:40.487Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","cluster":"counter-rn54g","authType":"IntegratedOAuth","serviceAccount":"counter-rn54g-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T15:59:40.487Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-rn54g","ray.io/group":"headgroup","ray.io/identifier":"counter-rn54g-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T15:59:40.487Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T15:59:40.487Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:59:40.487Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T15:59:40.487Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","enabled":true}
{"level":"info","ts":"2026-07-13T15:59:40.597Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","name":"counter-rn54g-head-kw6ml"}
{"level":"info","ts":"2026-07-13T15:59:40.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:40.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:40.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T15:59:40.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:40.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T15:59:40.597Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879"}
{"level":"info","ts":"2026-07-13T15:59:40.597Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","nodeType":"worker","rayStartParams":{"address":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T15:59:40.597Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","rayStartCmd":"ray start  --address=counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:59:40.597Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T15:59:40.597Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","enabled":true}
{"level":"info","ts":"2026-07-13T15:59:40.611Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","name":"counter-rn54g-small-group-worker-rdcgh"}
{"level":"info","ts":"2026-07-13T15:59:40.611Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","podName":"counter-rn54g-head-kw6ml","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:40.611Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","podName":"counter-rn54g-head-kw6ml","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:40.611Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-rn54g-head-svc"},"newHead":{"podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"}}
{"level":"info","ts":"2026-07-13T15:59:40.611Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1764b7fa-d458-473a-a297-f0178249e879","name":"counter-rn54g","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:59:40.626Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c41452ab-c348-4059-89c6-91266709328c"}
{"level":"info","ts":"2026-07-13T15:59:40.626Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c41452ab-c348-4059-89c6-91266709328c"}
{"level":"info","ts":"2026-07-13T15:59:40.626Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c41452ab-c348-4059-89c6-91266709328c","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:40.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c41452ab-c348-4059-89c6-91266709328c","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:40.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c41452ab-c348-4059-89c6-91266709328c","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:40.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c41452ab-c348-4059-89c6-91266709328c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:40.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c41452ab-c348-4059-89c6-91266709328c","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:40.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c41452ab-c348-4059-89c6-91266709328c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:40.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c41452ab-c348-4059-89c6-91266709328c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:40.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c41452ab-c348-4059-89c6-91266709328c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:40.626Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c41452ab-c348-4059-89c6-91266709328c","podName":"counter-rn54g-head-kw6ml","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:40.626Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c41452ab-c348-4059-89c6-91266709328c","podName":"counter-rn54g-head-kw6ml","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:40.626Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"828665ba-add6-4e4d-9860-aceb44930c04","rayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:40.626Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c41452ab-c348-4059-89c6-91266709328c","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-rn54g-head-svc"},"newHead":{"podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"}}
{"level":"info","ts":"2026-07-13T15:59:40.626Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"b126614d-1efa-4b90-8afc-9ecaa09e9443","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:40.626Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"a3a82c92-7ced-4447-8b8a-cdc1767ef65b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:40.626Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"828665ba-add6-4e4d-9860-aceb44930c04","rayCluster":"counter-rn54g","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:59:40.626Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"a3a82c92-7ced-4447-8b8a-cdc1767ef65b","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:40.627Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"a3a82c92-7ced-4447-8b8a-cdc1767ef65b","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:40.626Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c41452ab-c348-4059-89c6-91266709328c","name":"counter-rn54g","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:59:40.627Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"a3a82c92-7ced-4447-8b8a-cdc1767ef65b"}
{"level":"info","ts":"2026-07-13T15:59:40.627Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"a3a82c92-7ced-4447-8b8a-cdc1767ef65b","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T15:59:40.636Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c41452ab-c348-4059-89c6-91266709328c","name":"counter-rn54g","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-rn54g\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-x5qvg","name":"counter-rn54g"}}
{"level":"info","ts":"2026-07-13T15:59:40.636Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c41452ab-c348-4059-89c6-91266709328c"}
{"level":"error","ts":"2026-07-13T15:59:40.636Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"c41452ab-c348-4059-89c6-91266709328c","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-rn54g\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:40.636Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2bc03be1-c5d1-4aaf-a88c-0d5bbaec7a50"}
{"level":"info","ts":"2026-07-13T15:59:40.636Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2bc03be1-c5d1-4aaf-a88c-0d5bbaec7a50"}
{"level":"info","ts":"2026-07-13T15:59:40.636Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2bc03be1-c5d1-4aaf-a88c-0d5bbaec7a50","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:40.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2bc03be1-c5d1-4aaf-a88c-0d5bbaec7a50","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:40.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2bc03be1-c5d1-4aaf-a88c-0d5bbaec7a50","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:40.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2bc03be1-c5d1-4aaf-a88c-0d5bbaec7a50","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:40.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2bc03be1-c5d1-4aaf-a88c-0d5bbaec7a50","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:40.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2bc03be1-c5d1-4aaf-a88c-0d5bbaec7a50","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:40.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2bc03be1-c5d1-4aaf-a88c-0d5bbaec7a50","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:40.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2bc03be1-c5d1-4aaf-a88c-0d5bbaec7a50","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:40.636Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2bc03be1-c5d1-4aaf-a88c-0d5bbaec7a50","podName":"counter-rn54g-head-kw6ml","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:40.636Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2bc03be1-c5d1-4aaf-a88c-0d5bbaec7a50","podName":"counter-rn54g-head-kw6ml","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:40.636Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2bc03be1-c5d1-4aaf-a88c-0d5bbaec7a50","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:59:40.636Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2bc03be1-c5d1-4aaf-a88c-0d5bbaec7a50","seconds":300}
{"level":"info","ts":"2026-07-13T15:59:40.641Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"a08aedf8-de8c-4ec7-a5b0-1c87ba3231fe"}
{"level":"info","ts":"2026-07-13T15:59:40.641Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"a08aedf8-de8c-4ec7-a5b0-1c87ba3231fe"}
{"level":"info","ts":"2026-07-13T15:59:40.642Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"a08aedf8-de8c-4ec7-a5b0-1c87ba3231fe","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:40.642Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"a08aedf8-de8c-4ec7-a5b0-1c87ba3231fe","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:40.642Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"a08aedf8-de8c-4ec7-a5b0-1c87ba3231fe","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:40.642Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"a08aedf8-de8c-4ec7-a5b0-1c87ba3231fe","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:40.642Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"a08aedf8-de8c-4ec7-a5b0-1c87ba3231fe","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:40.642Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"a08aedf8-de8c-4ec7-a5b0-1c87ba3231fe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:40.642Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"a08aedf8-de8c-4ec7-a5b0-1c87ba3231fe","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:40.642Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"a08aedf8-de8c-4ec7-a5b0-1c87ba3231fe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:40.642Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"a08aedf8-de8c-4ec7-a5b0-1c87ba3231fe","podName":"counter-rn54g-head-kw6ml","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:40.642Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"a08aedf8-de8c-4ec7-a5b0-1c87ba3231fe","podName":"counter-rn54g-head-kw6ml","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:40.642Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"a08aedf8-de8c-4ec7-a5b0-1c87ba3231fe","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:59:40.642Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"a08aedf8-de8c-4ec7-a5b0-1c87ba3231fe","seconds":300}
{"level":"info","ts":"2026-07-13T15:59:40.642Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"b126614d-1efa-4b90-8afc-9ecaa09e9443","name":"counter-rn54g-head"}
{"level":"info","ts":"2026-07-13T15:59:40.646Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"8355aa48-1589-4f00-a806-b6c77f4b0e3b"}
{"level":"info","ts":"2026-07-13T15:59:40.647Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"8355aa48-1589-4f00-a806-b6c77f4b0e3b"}
{"level":"info","ts":"2026-07-13T15:59:40.647Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"8355aa48-1589-4f00-a806-b6c77f4b0e3b","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:40.647Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"8355aa48-1589-4f00-a806-b6c77f4b0e3b","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:40.647Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"8355aa48-1589-4f00-a806-b6c77f4b0e3b","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:40.647Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"8355aa48-1589-4f00-a806-b6c77f4b0e3b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:40.647Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"8355aa48-1589-4f00-a806-b6c77f4b0e3b","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:40.647Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"8355aa48-1589-4f00-a806-b6c77f4b0e3b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:40.647Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"8355aa48-1589-4f00-a806-b6c77f4b0e3b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:40.647Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"8355aa48-1589-4f00-a806-b6c77f4b0e3b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:40.647Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"8355aa48-1589-4f00-a806-b6c77f4b0e3b","podName":"counter-rn54g-head-kw6ml","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:40.647Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"8355aa48-1589-4f00-a806-b6c77f4b0e3b","podName":"counter-rn54g-head-kw6ml","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:40.647Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"8355aa48-1589-4f00-a806-b6c77f4b0e3b","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:59:40.647Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"8355aa48-1589-4f00-a806-b6c77f4b0e3b","name":"counter-rn54g","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:59:40.721Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"b126614d-1efa-4b90-8afc-9ecaa09e9443","name":"counter-rn54g-workers"}
{"level":"info","ts":"2026-07-13T15:59:40.721Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"b126614d-1efa-4b90-8afc-9ecaa09e9443","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3fa138c4-4b0a-49e0-83d3-3cec5e0e773f"}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3fa138c4-4b0a-49e0-83d3-3cec5e0e773f"}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3fa138c4-4b0a-49e0-83d3-3cec5e0e773f","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"b322ecfa-66d6-4fae-906c-caeab6674d74","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3fa138c4-4b0a-49e0-83d3-3cec5e0e773f","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3fa138c4-4b0a-49e0-83d3-3cec5e0e773f","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3fa138c4-4b0a-49e0-83d3-3cec5e0e773f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e7d7b263-0fc1-40f2-bf9d-fd47ffcb2e0a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"2ab98c84-b537-4cb9-af1d-7a1e77ed9185","rayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3fa138c4-4b0a-49e0-83d3-3cec5e0e773f","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3fa138c4-4b0a-49e0-83d3-3cec5e0e773f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3fa138c4-4b0a-49e0-83d3-3cec5e0e773f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3fa138c4-4b0a-49e0-83d3-3cec5e0e773f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e7d7b263-0fc1-40f2-bf9d-fd47ffcb2e0a","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e7d7b263-0fc1-40f2-bf9d-fd47ffcb2e0a","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3fa138c4-4b0a-49e0-83d3-3cec5e0e773f","podName":"counter-rn54g-head-kw6ml","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"2ab98c84-b537-4cb9-af1d-7a1e77ed9185","rayCluster":"counter-rn54g","podIPs":[]}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3fa138c4-4b0a-49e0-83d3-3cec5e0e773f","podName":"counter-rn54g-head-kw6ml","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e7d7b263-0fc1-40f2-bf9d-fd47ffcb2e0a"}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e7d7b263-0fc1-40f2-bf9d-fd47ffcb2e0a","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3fa138c4-4b0a-49e0-83d3-3cec5e0e773f","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:59:40.722Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3fa138c4-4b0a-49e0-83d3-3cec5e0e773f","name":"counter-rn54g","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:59:40.729Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3fa138c4-4b0a-49e0-83d3-3cec5e0e773f","name":"counter-rn54g","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-rn54g\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-x5qvg","name":"counter-rn54g"}}
{"level":"info","ts":"2026-07-13T15:59:40.729Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3fa138c4-4b0a-49e0-83d3-3cec5e0e773f"}
{"level":"error","ts":"2026-07-13T15:59:40.729Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"3fa138c4-4b0a-49e0-83d3-3cec5e0e773f","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-rn54g\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:59:40.735Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"700c8e69-50ea-45c8-ac9c-8c762a2b61b3"}
{"level":"info","ts":"2026-07-13T15:59:40.735Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"700c8e69-50ea-45c8-ac9c-8c762a2b61b3"}
{"level":"info","ts":"2026-07-13T15:59:40.735Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"700c8e69-50ea-45c8-ac9c-8c762a2b61b3","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:40.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"700c8e69-50ea-45c8-ac9c-8c762a2b61b3","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:40.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"700c8e69-50ea-45c8-ac9c-8c762a2b61b3","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:40.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"700c8e69-50ea-45c8-ac9c-8c762a2b61b3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:40.735Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"b322ecfa-66d6-4fae-906c-caeab6674d74","name":"counter-rn54g-head"}
{"level":"info","ts":"2026-07-13T15:59:40.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"700c8e69-50ea-45c8-ac9c-8c762a2b61b3","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:40.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"700c8e69-50ea-45c8-ac9c-8c762a2b61b3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:40.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"700c8e69-50ea-45c8-ac9c-8c762a2b61b3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:40.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"700c8e69-50ea-45c8-ac9c-8c762a2b61b3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:40.735Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"700c8e69-50ea-45c8-ac9c-8c762a2b61b3","podName":"counter-rn54g-head-kw6ml","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:40.735Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"700c8e69-50ea-45c8-ac9c-8c762a2b61b3","podName":"counter-rn54g-head-kw6ml","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:40.735Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"700c8e69-50ea-45c8-ac9c-8c762a2b61b3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:59:40.735Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"700c8e69-50ea-45c8-ac9c-8c762a2b61b3","seconds":300}
{"level":"info","ts":"2026-07-13T15:59:40.746Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"b322ecfa-66d6-4fae-906c-caeab6674d74","name":"counter-rn54g-workers"}
{"level":"info","ts":"2026-07-13T15:59:40.746Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"b322ecfa-66d6-4fae-906c-caeab6674d74","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:41.048Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"f8cd6c34-db70-4c7a-b5e6-9150aad77209"}
{"level":"info","ts":"2026-07-13T15:59:41.048Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"f8cd6c34-db70-4c7a-b5e6-9150aad77209"}
{"level":"info","ts":"2026-07-13T15:59:41.048Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"f8cd6c34-db70-4c7a-b5e6-9150aad77209","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:41.048Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"f8cd6c34-db70-4c7a-b5e6-9150aad77209","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:41.049Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"f8cd6c34-db70-4c7a-b5e6-9150aad77209","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:41.049Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"f8cd6c34-db70-4c7a-b5e6-9150aad77209","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:41.049Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"f8cd6c34-db70-4c7a-b5e6-9150aad77209","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:41.049Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"f8cd6c34-db70-4c7a-b5e6-9150aad77209","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:41.049Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"f8cd6c34-db70-4c7a-b5e6-9150aad77209","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:41.049Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"f8cd6c34-db70-4c7a-b5e6-9150aad77209","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:41.049Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"f8cd6c34-db70-4c7a-b5e6-9150aad77209","podName":"counter-rn54g-small-group-worker-rdcgh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:41.049Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"f8cd6c34-db70-4c7a-b5e6-9150aad77209","podName":"counter-rn54g-small-group-worker-rdcgh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:41.049Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"f8cd6c34-db70-4c7a-b5e6-9150aad77209","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:59:41.049Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"f8cd6c34-db70-4c7a-b5e6-9150aad77209","seconds":300}
{"level":"info","ts":"2026-07-13T15:59:41.063Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9a8bb073-b0fd-4615-8f03-af20d502e5f2"}
{"level":"info","ts":"2026-07-13T15:59:41.063Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9a8bb073-b0fd-4615-8f03-af20d502e5f2"}
{"level":"info","ts":"2026-07-13T15:59:41.063Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9a8bb073-b0fd-4615-8f03-af20d502e5f2","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:41.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9a8bb073-b0fd-4615-8f03-af20d502e5f2","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:41.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9a8bb073-b0fd-4615-8f03-af20d502e5f2","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:41.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9a8bb073-b0fd-4615-8f03-af20d502e5f2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:41.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9a8bb073-b0fd-4615-8f03-af20d502e5f2","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:41.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9a8bb073-b0fd-4615-8f03-af20d502e5f2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:41.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9a8bb073-b0fd-4615-8f03-af20d502e5f2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:41.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9a8bb073-b0fd-4615-8f03-af20d502e5f2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:41.063Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9a8bb073-b0fd-4615-8f03-af20d502e5f2","podName":"counter-rn54g-head-kw6ml","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:41.064Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9a8bb073-b0fd-4615-8f03-af20d502e5f2","podName":"counter-rn54g-head-kw6ml","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:41.064Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9a8bb073-b0fd-4615-8f03-af20d502e5f2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:59:41.064Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9a8bb073-b0fd-4615-8f03-af20d502e5f2","seconds":300}
{"level":"info","ts":"2026-07-13T15:59:41.323Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"16d646c7-c5ac-434e-b0d2-5694a6707a46","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:41.323Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"16d646c7-c5ac-434e-b0d2-5694a6707a46","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:41.323Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"16d646c7-c5ac-434e-b0d2-5694a6707a46","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:41.323Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"16d646c7-c5ac-434e-b0d2-5694a6707a46"}
{"level":"info","ts":"2026-07-13T15:59:41.323Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"16d646c7-c5ac-434e-b0d2-5694a6707a46","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T15:59:41.376Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2271cbf7-4872-45f6-9ef1-27b55e028271"}
{"level":"info","ts":"2026-07-13T15:59:41.376Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2271cbf7-4872-45f6-9ef1-27b55e028271"}
{"level":"info","ts":"2026-07-13T15:59:41.376Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2271cbf7-4872-45f6-9ef1-27b55e028271","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:41.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2271cbf7-4872-45f6-9ef1-27b55e028271","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:41.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2271cbf7-4872-45f6-9ef1-27b55e028271","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:41.377Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2271cbf7-4872-45f6-9ef1-27b55e028271","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:41.377Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2271cbf7-4872-45f6-9ef1-27b55e028271","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:41.377Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2271cbf7-4872-45f6-9ef1-27b55e028271","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:41.377Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2271cbf7-4872-45f6-9ef1-27b55e028271","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:41.377Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2271cbf7-4872-45f6-9ef1-27b55e028271","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:41.377Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2271cbf7-4872-45f6-9ef1-27b55e028271","podName":"counter-rn54g-small-group-worker-rdcgh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:41.377Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2271cbf7-4872-45f6-9ef1-27b55e028271","podName":"counter-rn54g-small-group-worker-rdcgh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:41.377Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2271cbf7-4872-45f6-9ef1-27b55e028271","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:59:41.377Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2271cbf7-4872-45f6-9ef1-27b55e028271","seconds":300}
{"level":"info","ts":"2026-07-13T15:59:42.122Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"6e72fe75-092c-4633-a123-fb915c71481e"}
{"level":"info","ts":"2026-07-13T15:59:42.122Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"6e72fe75-092c-4633-a123-fb915c71481e"}
{"level":"info","ts":"2026-07-13T15:59:42.122Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"6e72fe75-092c-4633-a123-fb915c71481e","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:42.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"6e72fe75-092c-4633-a123-fb915c71481e","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:42.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"6e72fe75-092c-4633-a123-fb915c71481e","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:42.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"6e72fe75-092c-4633-a123-fb915c71481e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:42.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"6e72fe75-092c-4633-a123-fb915c71481e","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:42.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"6e72fe75-092c-4633-a123-fb915c71481e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:42.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"6e72fe75-092c-4633-a123-fb915c71481e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:42.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"6e72fe75-092c-4633-a123-fb915c71481e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:42.122Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"6e72fe75-092c-4633-a123-fb915c71481e","podName":"counter-rn54g-small-group-worker-rdcgh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:42.122Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"6e72fe75-092c-4633-a123-fb915c71481e","podName":"counter-rn54g-small-group-worker-rdcgh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:42.122Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"6e72fe75-092c-4633-a123-fb915c71481e","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"newHead":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"}}
{"level":"info","ts":"2026-07-13T15:59:42.122Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"6e72fe75-092c-4633-a123-fb915c71481e","name":"counter-rn54g","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:59:42.134Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"a2319e88-8e8b-48fd-8961-e4092fc154af","rayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:42.134Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"5507cb3d-d111-43fd-bc31-8f1f2f4a29cc","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:42.134Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"a2319e88-8e8b-48fd-8961-e4092fc154af","pod":"counter-rn54g-head-kw6ml","ip":"10.133.0.28"}
{"level":"info","ts":"2026-07-13T15:59:42.134Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"a2319e88-8e8b-48fd-8961-e4092fc154af","pod":"counter-rn54g-small-group-worker-rdcgh","ip":"10.134.0.31"}
{"level":"info","ts":"2026-07-13T15:59:42.134Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"3e41fbbb-e1e7-43e5-8a2e-5ffdbbbdbcf6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:42.134Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"3e41fbbb-e1e7-43e5-8a2e-5ffdbbbdbcf6","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:42.134Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"3e41fbbb-e1e7-43e5-8a2e-5ffdbbbdbcf6","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:42.134Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"3e41fbbb-e1e7-43e5-8a2e-5ffdbbbdbcf6"}
{"level":"info","ts":"2026-07-13T15:59:42.134Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"3e41fbbb-e1e7-43e5-8a2e-5ffdbbbdbcf6","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T15:59:42.145Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"5507cb3d-d111-43fd-bc31-8f1f2f4a29cc","name":"counter-rn54g-head"}
{"level":"info","ts":"2026-07-13T15:59:42.146Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"a2319e88-8e8b-48fd-8961-e4092fc154af","certificate":"ray-head-cert-counter-rn54g","dnsNames":["counter-rn54g-head-svc","localhost","counter-rn54g-head-svc.test-ns-x5qvg.svc","counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local"],"ipAddresses":["10.133.0.28","10.134.0.31","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:59:42.156Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"a2319e88-8e8b-48fd-8961-e4092fc154af","certificate":"ray-worker-cert-counter-rn54g","dnsNames":["counter-rn54g-worker-svc","localhost","counter-rn54g-small-group","counter-rn54g-worker-svc.test-ns-x5qvg.svc","counter-rn54g-worker-svc.test-ns-x5qvg.svc.cluster.local","*.counter-rn54g-worker-svc.test-ns-x5qvg.svc","*.counter-rn54g-worker-svc.test-ns-x5qvg.svc.cluster.local","*-worker-*.test-ns-x5qvg.svc","*-worker-*.test-ns-x5qvg.svc.cluster.local"],"ipAddresses":["10.133.0.28","10.134.0.31","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T15:59:42.156Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"a2319e88-8e8b-48fd-8961-e4092fc154af","rayCluster":"counter-rn54g","podIPs":["10.133.0.28","10.134.0.31"]}
{"level":"info","ts":"2026-07-13T15:59:42.158Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"5507cb3d-d111-43fd-bc31-8f1f2f4a29cc","name":"counter-rn54g-workers"}
{"level":"info","ts":"2026-07-13T15:59:42.158Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"5507cb3d-d111-43fd-bc31-8f1f2f4a29cc","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:44.134Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"46ac82c8-ead8-48a7-bc31-6b71a01bb0cb"}
{"level":"info","ts":"2026-07-13T15:59:44.134Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"46ac82c8-ead8-48a7-bc31-6b71a01bb0cb"}
{"level":"info","ts":"2026-07-13T15:59:44.134Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"46ac82c8-ead8-48a7-bc31-6b71a01bb0cb","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:44.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"46ac82c8-ead8-48a7-bc31-6b71a01bb0cb","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:44.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"46ac82c8-ead8-48a7-bc31-6b71a01bb0cb","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:44.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"46ac82c8-ead8-48a7-bc31-6b71a01bb0cb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:44.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"46ac82c8-ead8-48a7-bc31-6b71a01bb0cb","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:44.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"46ac82c8-ead8-48a7-bc31-6b71a01bb0cb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:44.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"46ac82c8-ead8-48a7-bc31-6b71a01bb0cb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:44.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"46ac82c8-ead8-48a7-bc31-6b71a01bb0cb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:44.134Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"46ac82c8-ead8-48a7-bc31-6b71a01bb0cb","podName":"counter-rn54g-head-kw6ml","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:44.134Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"46ac82c8-ead8-48a7-bc31-6b71a01bb0cb","podName":"counter-rn54g-head-kw6ml","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:44.135Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"46ac82c8-ead8-48a7-bc31-6b71a01bb0cb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:59:44.135Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"46ac82c8-ead8-48a7-bc31-6b71a01bb0cb","seconds":300}
{"level":"info","ts":"2026-07-13T15:59:44.323Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"04132c11-c24d-4379-88df-0161b5f4a154","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:44.323Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"04132c11-c24d-4379-88df-0161b5f4a154","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:44.323Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"04132c11-c24d-4379-88df-0161b5f4a154","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:44.324Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"04132c11-c24d-4379-88df-0161b5f4a154"}
{"level":"info","ts":"2026-07-13T15:59:44.324Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"04132c11-c24d-4379-88df-0161b5f4a154","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T15:59:46.398Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53f4ed10-2784-4c1e-8d88-cfa58e0cd2bd"}
{"level":"info","ts":"2026-07-13T15:59:46.398Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53f4ed10-2784-4c1e-8d88-cfa58e0cd2bd"}
{"level":"info","ts":"2026-07-13T15:59:46.398Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53f4ed10-2784-4c1e-8d88-cfa58e0cd2bd","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:46.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53f4ed10-2784-4c1e-8d88-cfa58e0cd2bd","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:46.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53f4ed10-2784-4c1e-8d88-cfa58e0cd2bd","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:46.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53f4ed10-2784-4c1e-8d88-cfa58e0cd2bd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:46.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53f4ed10-2784-4c1e-8d88-cfa58e0cd2bd","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:46.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53f4ed10-2784-4c1e-8d88-cfa58e0cd2bd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:46.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53f4ed10-2784-4c1e-8d88-cfa58e0cd2bd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:46.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53f4ed10-2784-4c1e-8d88-cfa58e0cd2bd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:46.398Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53f4ed10-2784-4c1e-8d88-cfa58e0cd2bd","podName":"counter-rn54g-small-group-worker-rdcgh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:46.398Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53f4ed10-2784-4c1e-8d88-cfa58e0cd2bd","podName":"counter-rn54g-small-group-worker-rdcgh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:46.399Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53f4ed10-2784-4c1e-8d88-cfa58e0cd2bd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:59:46.399Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"53f4ed10-2784-4c1e-8d88-cfa58e0cd2bd","seconds":300}
{"level":"info","ts":"2026-07-13T15:59:47.144Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"66963031-f57c-473e-95e4-5fc2e263d2b3"}
{"level":"info","ts":"2026-07-13T15:59:47.144Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"66963031-f57c-473e-95e4-5fc2e263d2b3"}
{"level":"info","ts":"2026-07-13T15:59:47.144Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"66963031-f57c-473e-95e4-5fc2e263d2b3","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:47.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"66963031-f57c-473e-95e4-5fc2e263d2b3","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:47.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"66963031-f57c-473e-95e4-5fc2e263d2b3","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:47.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"66963031-f57c-473e-95e4-5fc2e263d2b3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:47.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"66963031-f57c-473e-95e4-5fc2e263d2b3","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:47.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"66963031-f57c-473e-95e4-5fc2e263d2b3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:47.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"66963031-f57c-473e-95e4-5fc2e263d2b3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:47.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"66963031-f57c-473e-95e4-5fc2e263d2b3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:47.145Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"66963031-f57c-473e-95e4-5fc2e263d2b3","podName":"counter-rn54g-small-group-worker-rdcgh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:47.145Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"66963031-f57c-473e-95e4-5fc2e263d2b3","podName":"counter-rn54g-small-group-worker-rdcgh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:47.145Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"66963031-f57c-473e-95e4-5fc2e263d2b3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:59:47.145Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"66963031-f57c-473e-95e4-5fc2e263d2b3","seconds":300}
{"level":"info","ts":"2026-07-13T15:59:47.324Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"7152ea2f-ecd4-4221-a910-69c7e20009d5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:47.324Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"7152ea2f-ecd4-4221-a910-69c7e20009d5","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:47.324Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"7152ea2f-ecd4-4221-a910-69c7e20009d5","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:47.325Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"7152ea2f-ecd4-4221-a910-69c7e20009d5"}
{"level":"info","ts":"2026-07-13T15:59:47.325Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"7152ea2f-ecd4-4221-a910-69c7e20009d5","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T15:59:47.405Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1a973e3d-6375-4218-a275-f58074d9fdca"}
{"level":"info","ts":"2026-07-13T15:59:47.405Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1a973e3d-6375-4218-a275-f58074d9fdca"}
{"level":"info","ts":"2026-07-13T15:59:47.405Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1a973e3d-6375-4218-a275-f58074d9fdca","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:47.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1a973e3d-6375-4218-a275-f58074d9fdca","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:47.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1a973e3d-6375-4218-a275-f58074d9fdca","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:47.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1a973e3d-6375-4218-a275-f58074d9fdca","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:47.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1a973e3d-6375-4218-a275-f58074d9fdca","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:47.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1a973e3d-6375-4218-a275-f58074d9fdca","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:47.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1a973e3d-6375-4218-a275-f58074d9fdca","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:47.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1a973e3d-6375-4218-a275-f58074d9fdca","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:47.405Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1a973e3d-6375-4218-a275-f58074d9fdca","podName":"counter-rn54g-head-kw6ml","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:47.405Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1a973e3d-6375-4218-a275-f58074d9fdca","podName":"counter-rn54g-head-kw6ml","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T15:59:47.405Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1a973e3d-6375-4218-a275-f58074d9fdca","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:59:47.405Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"1a973e3d-6375-4218-a275-f58074d9fdca","seconds":300}
{"level":"info","ts":"2026-07-13T15:59:48.151Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5c1415ed-51fb-496b-ab92-7676a030b6a7"}
{"level":"info","ts":"2026-07-13T15:59:48.151Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5c1415ed-51fb-496b-ab92-7676a030b6a7"}
{"level":"info","ts":"2026-07-13T15:59:48.151Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5c1415ed-51fb-496b-ab92-7676a030b6a7","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:48.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5c1415ed-51fb-496b-ab92-7676a030b6a7","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:48.152Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5c1415ed-51fb-496b-ab92-7676a030b6a7","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:48.152Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5c1415ed-51fb-496b-ab92-7676a030b6a7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:48.152Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5c1415ed-51fb-496b-ab92-7676a030b6a7","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:48.152Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5c1415ed-51fb-496b-ab92-7676a030b6a7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:48.152Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5c1415ed-51fb-496b-ab92-7676a030b6a7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:48.152Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5c1415ed-51fb-496b-ab92-7676a030b6a7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:48.152Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5c1415ed-51fb-496b-ab92-7676a030b6a7","podName":"counter-rn54g-head-kw6ml","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:59:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:59:48.152Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5c1415ed-51fb-496b-ab92-7676a030b6a7","podName":"counter-rn54g-head-kw6ml","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:59:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:59:48.152Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5c1415ed-51fb-496b-ab92-7676a030b6a7","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:59:48.152Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5c1415ed-51fb-496b-ab92-7676a030b6a7","name":"counter-rn54g","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:59:48.164Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"4d3de952-5c6b-40a6-b6ea-399c5a2ce76b","rayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:48.164Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"c2a5f5fd-8926-4975-b9a2-c5452123f3b9","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:48.165Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"82aa73cd-6873-4fc5-8aae-05132f79634a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:48.165Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"82aa73cd-6873-4fc5-8aae-05132f79634a","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:48.165Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"82aa73cd-6873-4fc5-8aae-05132f79634a","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:48.165Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"4d3de952-5c6b-40a6-b6ea-399c5a2ce76b","pod":"counter-rn54g-head-kw6ml","ip":"10.133.0.28"}
{"level":"info","ts":"2026-07-13T15:59:48.165Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"4d3de952-5c6b-40a6-b6ea-399c5a2ce76b","pod":"counter-rn54g-small-group-worker-rdcgh","ip":"10.134.0.31"}
{"level":"info","ts":"2026-07-13T15:59:48.165Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"4d3de952-5c6b-40a6-b6ea-399c5a2ce76b","rayCluster":"counter-rn54g","podIPs":["10.133.0.28","10.134.0.31"]}
{"level":"info","ts":"2026-07-13T15:59:48.165Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"82aa73cd-6873-4fc5-8aae-05132f79634a"}
{"level":"info","ts":"2026-07-13T15:59:48.165Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"82aa73cd-6873-4fc5-8aae-05132f79634a","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T15:59:48.174Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"c2a5f5fd-8926-4975-b9a2-c5452123f3b9","name":"counter-rn54g-head"}
{"level":"info","ts":"2026-07-13T15:59:48.185Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"c2a5f5fd-8926-4975-b9a2-c5452123f3b9","name":"counter-rn54g-workers"}
{"level":"info","ts":"2026-07-13T15:59:48.186Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"c2a5f5fd-8926-4975-b9a2-c5452123f3b9","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:50.165Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2289946a-9a47-44b7-a494-44711ea0e109"}
{"level":"info","ts":"2026-07-13T15:59:50.165Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2289946a-9a47-44b7-a494-44711ea0e109"}
{"level":"info","ts":"2026-07-13T15:59:50.165Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2289946a-9a47-44b7-a494-44711ea0e109","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:50.165Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2289946a-9a47-44b7-a494-44711ea0e109","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:50.165Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2289946a-9a47-44b7-a494-44711ea0e109","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:50.165Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2289946a-9a47-44b7-a494-44711ea0e109","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:50.165Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2289946a-9a47-44b7-a494-44711ea0e109","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:50.165Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2289946a-9a47-44b7-a494-44711ea0e109","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:50.165Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2289946a-9a47-44b7-a494-44711ea0e109","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:50.165Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2289946a-9a47-44b7-a494-44711ea0e109","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:50.165Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2289946a-9a47-44b7-a494-44711ea0e109","podName":"counter-rn54g-head-kw6ml","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:59:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:59:50.165Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2289946a-9a47-44b7-a494-44711ea0e109","podName":"counter-rn54g-head-kw6ml","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:59:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:59:50.165Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2289946a-9a47-44b7-a494-44711ea0e109","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:59:50.165Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"2289946a-9a47-44b7-a494-44711ea0e109","seconds":300}
{"level":"info","ts":"2026-07-13T15:59:50.337Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"571f0622-2ae5-4a01-9d32-d41507a0d9a0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:50.337Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"571f0622-2ae5-4a01-9d32-d41507a0d9a0","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:50.337Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"571f0622-2ae5-4a01-9d32-d41507a0d9a0","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:50.338Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"571f0622-2ae5-4a01-9d32-d41507a0d9a0"}
{"level":"info","ts":"2026-07-13T15:59:50.338Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"571f0622-2ae5-4a01-9d32-d41507a0d9a0","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T15:59:53.338Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"19845525-b27b-4846-b758-d4e5c46498df","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:53.338Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"19845525-b27b-4846-b758-d4e5c46498df","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:53.338Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"19845525-b27b-4846-b758-d4e5c46498df","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:53.339Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"19845525-b27b-4846-b758-d4e5c46498df"}
{"level":"info","ts":"2026-07-13T15:59:53.339Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"19845525-b27b-4846-b758-d4e5c46498df","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T15:59:56.340Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"b0ef9206-194f-4c1e-8f5c-61e58b78810a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:56.340Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"b0ef9206-194f-4c1e-8f5c-61e58b78810a","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:56.340Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"b0ef9206-194f-4c1e-8f5c-61e58b78810a","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:56.340Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"b0ef9206-194f-4c1e-8f5c-61e58b78810a"}
{"level":"info","ts":"2026-07-13T15:59:56.340Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"b0ef9206-194f-4c1e-8f5c-61e58b78810a","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T15:59:57.445Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"dbc007dd-e821-4004-be41-0ecdc1557987"}
{"level":"info","ts":"2026-07-13T15:59:57.445Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"dbc007dd-e821-4004-be41-0ecdc1557987"}
{"level":"info","ts":"2026-07-13T15:59:57.445Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"dbc007dd-e821-4004-be41-0ecdc1557987","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:57.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"dbc007dd-e821-4004-be41-0ecdc1557987","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:57.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"dbc007dd-e821-4004-be41-0ecdc1557987","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:57.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"dbc007dd-e821-4004-be41-0ecdc1557987","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:57.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"dbc007dd-e821-4004-be41-0ecdc1557987","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:57.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"dbc007dd-e821-4004-be41-0ecdc1557987","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:57.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"dbc007dd-e821-4004-be41-0ecdc1557987","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:57.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"dbc007dd-e821-4004-be41-0ecdc1557987","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:57.445Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"dbc007dd-e821-4004-be41-0ecdc1557987","podName":"counter-rn54g-head-kw6ml","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:59:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:59:57.445Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"dbc007dd-e821-4004-be41-0ecdc1557987","podName":"counter-rn54g-head-kw6ml","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:59:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:59:57.445Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"dbc007dd-e821-4004-be41-0ecdc1557987","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T15:59:57.445Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"dbc007dd-e821-4004-be41-0ecdc1557987","seconds":300}
{"level":"info","ts":"2026-07-13T15:59:58.452Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"d40e1099-1d9b-46b0-8991-499935febd65"}
{"level":"info","ts":"2026-07-13T15:59:58.452Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"d40e1099-1d9b-46b0-8991-499935febd65"}
{"level":"info","ts":"2026-07-13T15:59:58.452Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"d40e1099-1d9b-46b0-8991-499935febd65","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:58.453Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"d40e1099-1d9b-46b0-8991-499935febd65","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:58.453Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"d40e1099-1d9b-46b0-8991-499935febd65","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:58.453Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"d40e1099-1d9b-46b0-8991-499935febd65","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:58.453Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"d40e1099-1d9b-46b0-8991-499935febd65","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:58.453Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"d40e1099-1d9b-46b0-8991-499935febd65","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:58.453Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"d40e1099-1d9b-46b0-8991-499935febd65","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:58.453Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"d40e1099-1d9b-46b0-8991-499935febd65","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:58.453Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"d40e1099-1d9b-46b0-8991-499935febd65","podName":"counter-rn54g-head-kw6ml","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:59:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:59:58.453Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"d40e1099-1d9b-46b0-8991-499935febd65","podName":"counter-rn54g-head-kw6ml","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:59:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T15:59:58.453Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"d40e1099-1d9b-46b0-8991-499935febd65","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T15:59:58.453Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"d40e1099-1d9b-46b0-8991-499935febd65","name":"counter-rn54g","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:59:58.466Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"e004e8bc-5ab2-45d5-800c-75980ad4852f","rayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:58.466Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"7dca7254-fb18-4b13-ba0c-34bbf7a577f8","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:58.466Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"e004e8bc-5ab2-45d5-800c-75980ad4852f","pod":"counter-rn54g-head-kw6ml","ip":"10.133.0.28"}
{"level":"info","ts":"2026-07-13T15:59:58.466Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"e004e8bc-5ab2-45d5-800c-75980ad4852f","pod":"counter-rn54g-small-group-worker-rdcgh","ip":"10.134.0.31"}
{"level":"info","ts":"2026-07-13T15:59:58.466Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"8a8a1323-06bf-43a1-bab9-c929b59686c1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:58.466Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"8a8a1323-06bf-43a1-bab9-c929b59686c1","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:58.466Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"e004e8bc-5ab2-45d5-800c-75980ad4852f","rayCluster":"counter-rn54g","podIPs":["10.133.0.28","10.134.0.31"]}
{"level":"info","ts":"2026-07-13T15:59:58.466Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"8a8a1323-06bf-43a1-bab9-c929b59686c1","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:58.466Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"8a8a1323-06bf-43a1-bab9-c929b59686c1"}
{"level":"info","ts":"2026-07-13T15:59:58.466Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"8a8a1323-06bf-43a1-bab9-c929b59686c1","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T15:59:58.477Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"7dca7254-fb18-4b13-ba0c-34bbf7a577f8","name":"counter-rn54g-head"}
{"level":"info","ts":"2026-07-13T15:59:58.485Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"7dca7254-fb18-4b13-ba0c-34bbf7a577f8","name":"counter-rn54g-workers"}
{"level":"info","ts":"2026-07-13T15:59:58.485Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"7dca7254-fb18-4b13-ba0c-34bbf7a577f8","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:59.179Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5d4857f4-d58d-494e-8142-b7d961ce50a0"}
{"level":"info","ts":"2026-07-13T15:59:59.179Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5d4857f4-d58d-494e-8142-b7d961ce50a0"}
{"level":"info","ts":"2026-07-13T15:59:59.179Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5d4857f4-d58d-494e-8142-b7d961ce50a0","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T15:59:59.179Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5d4857f4-d58d-494e-8142-b7d961ce50a0","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T15:59:59.179Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5d4857f4-d58d-494e-8142-b7d961ce50a0","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:59.179Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5d4857f4-d58d-494e-8142-b7d961ce50a0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T15:59:59.179Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5d4857f4-d58d-494e-8142-b7d961ce50a0","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T15:59:59.179Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5d4857f4-d58d-494e-8142-b7d961ce50a0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:59.179Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5d4857f4-d58d-494e-8142-b7d961ce50a0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T15:59:59.179Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5d4857f4-d58d-494e-8142-b7d961ce50a0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T15:59:59.179Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5d4857f4-d58d-494e-8142-b7d961ce50a0","podName":"counter-rn54g-small-group-worker-rdcgh","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:59:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:59:59.179Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5d4857f4-d58d-494e-8142-b7d961ce50a0","podName":"counter-rn54g-small-group-worker-rdcgh","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:59:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T15:59:59.179Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5d4857f4-d58d-494e-8142-b7d961ce50a0","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:59:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T15:59:59.179Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5d4857f4-d58d-494e-8142-b7d961ce50a0","name":"counter-rn54g","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:59:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T15:59:59.192Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"029c1ab4-398f-4f6d-be7c-de2b12bfcf82","rayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:59.192Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"c88f8fad-afe9-4d19-b331-a1b003421ea6","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:59.192Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"029c1ab4-398f-4f6d-be7c-de2b12bfcf82","pod":"counter-rn54g-head-kw6ml","ip":"10.133.0.28"}
{"level":"info","ts":"2026-07-13T15:59:59.192Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"029c1ab4-398f-4f6d-be7c-de2b12bfcf82","pod":"counter-rn54g-small-group-worker-rdcgh","ip":"10.134.0.31"}
{"level":"info","ts":"2026-07-13T15:59:59.192Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"029c1ab4-398f-4f6d-be7c-de2b12bfcf82","rayCluster":"counter-rn54g","podIPs":["10.133.0.28","10.134.0.31"]}
{"level":"info","ts":"2026-07-13T15:59:59.192Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"decc5cf5-77db-4c29-be8b-bc5295123b2d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:59.192Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"decc5cf5-77db-4c29-be8b-bc5295123b2d","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:59.192Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"decc5cf5-77db-4c29-be8b-bc5295123b2d","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:59.192Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"decc5cf5-77db-4c29-be8b-bc5295123b2d"}
{"level":"info","ts":"2026-07-13T15:59:59.193Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"decc5cf5-77db-4c29-be8b-bc5295123b2d","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T15:59:59.203Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"c88f8fad-afe9-4d19-b331-a1b003421ea6","name":"counter-rn54g-head"}
{"level":"info","ts":"2026-07-13T15:59:59.214Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"c88f8fad-afe9-4d19-b331-a1b003421ea6","name":"counter-rn54g-workers"}
{"level":"info","ts":"2026-07-13T15:59:59.214Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"c88f8fad-afe9-4d19-b331-a1b003421ea6","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:59.340Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"6589137d-61d6-44ee-bdf0-e59e33bfee88","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T15:59:59.340Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"6589137d-61d6-44ee-bdf0-e59e33bfee88","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T15:59:59.340Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"6589137d-61d6-44ee-bdf0-e59e33bfee88","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T15:59:59.340Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"6589137d-61d6-44ee-bdf0-e59e33bfee88"}
{"level":"info","ts":"2026-07-13T15:59:59.340Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"6589137d-61d6-44ee-bdf0-e59e33bfee88","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T16:00:00.466Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"0a8d30fe-1c7d-4961-9fc7-bed466025c3c"}
{"level":"info","ts":"2026-07-13T16:00:00.466Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"0a8d30fe-1c7d-4961-9fc7-bed466025c3c"}
{"level":"info","ts":"2026-07-13T16:00:00.466Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"0a8d30fe-1c7d-4961-9fc7-bed466025c3c","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T16:00:00.466Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"0a8d30fe-1c7d-4961-9fc7-bed466025c3c","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:00:00.466Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"0a8d30fe-1c7d-4961-9fc7-bed466025c3c","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:00:00.466Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"0a8d30fe-1c7d-4961-9fc7-bed466025c3c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:00:00.466Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"0a8d30fe-1c7d-4961-9fc7-bed466025c3c","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:00:00.466Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"0a8d30fe-1c7d-4961-9fc7-bed466025c3c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:00:00.466Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"0a8d30fe-1c7d-4961-9fc7-bed466025c3c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:00:00.466Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"0a8d30fe-1c7d-4961-9fc7-bed466025c3c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:00:00.466Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"0a8d30fe-1c7d-4961-9fc7-bed466025c3c","podName":"counter-rn54g-small-group-worker-rdcgh","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:59:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:00:00.466Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"0a8d30fe-1c7d-4961-9fc7-bed466025c3c","podName":"counter-rn54g-small-group-worker-rdcgh","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T15:59:40Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:00:00.466Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"0a8d30fe-1c7d-4961-9fc7-bed466025c3c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:00:00.466Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"0a8d30fe-1c7d-4961-9fc7-bed466025c3c","seconds":300}
{"level":"info","ts":"2026-07-13T16:00:02.341Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"c8cdd720-4f42-47f3-9776-0487407f95e3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:02.341Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"c8cdd720-4f42-47f3-9776-0487407f95e3","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:02.341Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"c8cdd720-4f42-47f3-9776-0487407f95e3","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:02.341Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"c8cdd720-4f42-47f3-9776-0487407f95e3"}
{"level":"info","ts":"2026-07-13T16:00:02.341Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"c8cdd720-4f42-47f3-9776-0487407f95e3","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T16:00:05.343Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"c761315f-e9a6-4921-8c49-6a8050011b38","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:05.343Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"c761315f-e9a6-4921-8c49-6a8050011b38","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:05.343Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"c761315f-e9a6-4921-8c49-6a8050011b38","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:05.343Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"c761315f-e9a6-4921-8c49-6a8050011b38"}
{"level":"info","ts":"2026-07-13T16:00:05.343Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"c761315f-e9a6-4921-8c49-6a8050011b38","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T16:00:05.422Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"74df48c7-2a9d-426d-a741-6fa1afa4da07","rayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:05.422Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"74df48c7-2a9d-426d-a741-6fa1afa4da07","pod":"counter-rn54g-small-group-worker-rdcgh","ip":"10.134.0.31"}
{"level":"info","ts":"2026-07-13T16:00:05.422Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"74df48c7-2a9d-426d-a741-6fa1afa4da07","pod":"counter-rn54g-head-kw6ml","ip":"10.133.0.28"}
{"level":"info","ts":"2026-07-13T16:00:05.422Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"74df48c7-2a9d-426d-a741-6fa1afa4da07","rayCluster":"counter-rn54g","podIPs":["10.134.0.31","10.133.0.28"]}
{"level":"info","ts":"2026-07-13T16:00:08.344Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"73e84349-e4e3-4c91-bbe7-beda7f45bc27","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:08.344Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"73e84349-e4e3-4c91-bbe7-beda7f45bc27","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:08.344Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"73e84349-e4e3-4c91-bbe7-beda7f45bc27","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:08.344Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"73e84349-e4e3-4c91-bbe7-beda7f45bc27"}
{"level":"info","ts":"2026-07-13T16:00:08.344Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"73e84349-e4e3-4c91-bbe7-beda7f45bc27","RayCluster":"counter-rn54g","State":""}
{"level":"info","ts":"2026-07-13T16:00:09.477Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5e988450-7db7-4795-882a-2ad084b67b9d"}
{"level":"info","ts":"2026-07-13T16:00:09.477Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5e988450-7db7-4795-882a-2ad084b67b9d"}
{"level":"info","ts":"2026-07-13T16:00:09.477Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5e988450-7db7-4795-882a-2ad084b67b9d","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T16:00:09.477Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5e988450-7db7-4795-882a-2ad084b67b9d","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:00:09.477Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5e988450-7db7-4795-882a-2ad084b67b9d","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:00:09.477Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5e988450-7db7-4795-882a-2ad084b67b9d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:00:09.477Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5e988450-7db7-4795-882a-2ad084b67b9d","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:00:09.477Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5e988450-7db7-4795-882a-2ad084b67b9d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:00:09.477Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5e988450-7db7-4795-882a-2ad084b67b9d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:00:09.477Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5e988450-7db7-4795-882a-2ad084b67b9d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:00:09.478Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5e988450-7db7-4795-882a-2ad084b67b9d","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-13T16:00:09.478Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"5e988450-7db7-4795-882a-2ad084b67b9d","name":"counter-rn54g","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:59:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:59:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:00:09Z","stateTransitionTimes":{"ready":"2026-07-13T16:00:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:59:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:00:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:00:09.489Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"3202b033-36ed-4cd4-9d25-610089a32b6f","rayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:09.490Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"f81f2d11-ad38-48e0-ba06-0e3bf241ea8c","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:09.490Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"3202b033-36ed-4cd4-9d25-610089a32b6f","pod":"counter-rn54g-head-kw6ml","ip":"10.133.0.28"}
{"level":"info","ts":"2026-07-13T16:00:09.490Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"3202b033-36ed-4cd4-9d25-610089a32b6f","pod":"counter-rn54g-small-group-worker-rdcgh","ip":"10.134.0.31"}
{"level":"info","ts":"2026-07-13T16:00:09.490Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"d46d4b9e-c414-4a51-b590-3a41ca9fb13a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:09.490Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"3202b033-36ed-4cd4-9d25-610089a32b6f","rayCluster":"counter-rn54g","podIPs":["10.133.0.28","10.134.0.31"]}
{"level":"info","ts":"2026-07-13T16:00:09.490Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"d46d4b9e-c414-4a51-b590-3a41ca9fb13a","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:09.490Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"d46d4b9e-c414-4a51-b590-3a41ca9fb13a","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:09.490Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"d46d4b9e-c414-4a51-b590-3a41ca9fb13a"}
{"level":"info","ts":"2026-07-13T16:00:09.490Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"d46d4b9e-c414-4a51-b590-3a41ca9fb13a","head service name":"counter-rn54g-head-svc","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:00:09.490Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"d46d4b9e-c414-4a51-b590-3a41ca9fb13a","head service URL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-13T16:00:09.490Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"d46d4b9e-c414-4a51-b590-3a41ca9fb13a"}
{"level":"info","ts":"2026-07-13T16:00:09.490Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"d46d4b9e-c414-4a51-b590-3a41ca9fb13a","command":["if","!","ray","job","status","--address","http://counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","counter-pfwlp",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","--no-wait","--runtime-env-json","\"{\\\"env_vars\\\":{\\\"counter_name\\\":\\\"test_counter\\\"}}\"","--submission-id","counter-pfwlp","--","python /home/ray/jobs/counter.py",";","fi",";","ray","job","logs","--address","http://counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","--follow","counter-pfwlp"]}
{"level":"info","ts":"2026-07-13T16:00:09.503Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"d46d4b9e-c414-4a51-b590-3a41ca9fb13a","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-07-13T16:00:09.504Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"d46d4b9e-c414-4a51-b590-3a41ca9fb13a","SubmissionMode":"K8sJobMode","RayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:09.504Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"d46d4b9e-c414-4a51-b590-3a41ca9fb13a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","dashboardURL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T16:00:09.504Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"d46d4b9e-c414-4a51-b590-3a41ca9fb13a","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T16:00:09.508Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"f81f2d11-ad38-48e0-ba06-0e3bf241ea8c","name":"counter-rn54g-head"}
{"level":"info","ts":"2026-07-13T16:00:09.514Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"f2d54540-5cd6-4dba-b609-4aefa6b8470a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:09.514Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"f2d54540-5cd6-4dba-b609-4aefa6b8470a","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:09.514Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"f2d54540-5cd6-4dba-b609-4aefa6b8470a","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:09.514Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"f2d54540-5cd6-4dba-b609-4aefa6b8470a"}
{"level":"info","ts":"2026-07-13T16:00:09.514Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"f2d54540-5cd6-4dba-b609-4aefa6b8470a","head service name":"counter-rn54g-head-svc","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:00:09.514Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"f2d54540-5cd6-4dba-b609-4aefa6b8470a","head service URL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-13T16:00:09.514Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"f2d54540-5cd6-4dba-b609-4aefa6b8470a","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-07-13T16:00:09.514Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"f2d54540-5cd6-4dba-b609-4aefa6b8470a","SubmissionMode":"K8sJobMode","RayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:09.514Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"f2d54540-5cd6-4dba-b609-4aefa6b8470a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","dashboardURL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T16:00:09.514Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"f2d54540-5cd6-4dba-b609-4aefa6b8470a","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T16:00:09.524Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"f2d54540-5cd6-4dba-b609-4aefa6b8470a","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-13T16:00:09.524Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"f2d54540-5cd6-4dba-b609-4aefa6b8470a"}
{"level":"error","ts":"2026-07-13T16:00:09.524Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"f2d54540-5cd6-4dba-b609-4aefa6b8470a","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:09.524Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"42f82f0b-6d43-4c95-b62c-bc6f145fd30b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:09.524Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"42f82f0b-6d43-4c95-b62c-bc6f145fd30b","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:09.524Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"42f82f0b-6d43-4c95-b62c-bc6f145fd30b","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:09.524Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"42f82f0b-6d43-4c95-b62c-bc6f145fd30b"}
{"level":"info","ts":"2026-07-13T16:00:09.524Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"f81f2d11-ad38-48e0-ba06-0e3bf241ea8c","name":"counter-rn54g-workers"}
{"level":"info","ts":"2026-07-13T16:00:09.524Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"f81f2d11-ad38-48e0-ba06-0e3bf241ea8c","cluster":"counter-rn54g"}
{"level":"error","ts":"2026-07-13T16:00:09.533Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"42f82f0b-6d43-4c95-b62c-bc6f145fd30b","JobId":"counter-pfwlp","error":"Job counter-pfwlp does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:09.533Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"42f82f0b-6d43-4c95-b62c-bc6f145fd30b"}
{"level":"error","ts":"2026-07-13T16:00:09.533Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"42f82f0b-6d43-4c95-b62c-bc6f145fd30b","error":"Job counter-pfwlp does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:09.533Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"794eceb5-971c-408b-be27-64c39ddc6a0f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:09.533Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"794eceb5-971c-408b-be27-64c39ddc6a0f","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:09.533Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"794eceb5-971c-408b-be27-64c39ddc6a0f","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:09.533Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"794eceb5-971c-408b-be27-64c39ddc6a0f"}
{"level":"error","ts":"2026-07-13T16:00:09.539Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"794eceb5-971c-408b-be27-64c39ddc6a0f","JobId":"counter-pfwlp","error":"Job counter-pfwlp does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:09.539Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"794eceb5-971c-408b-be27-64c39ddc6a0f"}
{"level":"error","ts":"2026-07-13T16:00:09.539Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"794eceb5-971c-408b-be27-64c39ddc6a0f","error":"Job counter-pfwlp does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:09.543Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"65d40258-ab49-4467-abcb-0932ccd6040f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:09.543Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"65d40258-ab49-4467-abcb-0932ccd6040f","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:09.543Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"65d40258-ab49-4467-abcb-0932ccd6040f","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:09.543Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"65d40258-ab49-4467-abcb-0932ccd6040f"}
{"level":"error","ts":"2026-07-13T16:00:09.548Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"65d40258-ab49-4467-abcb-0932ccd6040f","JobId":"counter-pfwlp","error":"Job counter-pfwlp does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:09.548Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"65d40258-ab49-4467-abcb-0932ccd6040f"}
{"level":"error","ts":"2026-07-13T16:00:09.548Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"65d40258-ab49-4467-abcb-0932ccd6040f","error":"Job counter-pfwlp does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:09.588Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"3efa26a3-b6a3-45a8-9d13-33af403e8b2f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:09.589Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"3efa26a3-b6a3-45a8-9d13-33af403e8b2f","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:09.589Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"3efa26a3-b6a3-45a8-9d13-33af403e8b2f","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:09.589Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"3efa26a3-b6a3-45a8-9d13-33af403e8b2f"}
{"level":"error","ts":"2026-07-13T16:00:09.593Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"3efa26a3-b6a3-45a8-9d13-33af403e8b2f","JobId":"counter-pfwlp","error":"Job counter-pfwlp does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:09.593Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"3efa26a3-b6a3-45a8-9d13-33af403e8b2f"}
{"level":"error","ts":"2026-07-13T16:00:09.593Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"3efa26a3-b6a3-45a8-9d13-33af403e8b2f","error":"Job counter-pfwlp does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:09.673Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e5ea9439-349c-407f-80b8-05926b4af325","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:09.673Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e5ea9439-349c-407f-80b8-05926b4af325","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:09.673Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e5ea9439-349c-407f-80b8-05926b4af325","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:09.673Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e5ea9439-349c-407f-80b8-05926b4af325"}
{"level":"error","ts":"2026-07-13T16:00:09.677Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e5ea9439-349c-407f-80b8-05926b4af325","JobId":"counter-pfwlp","error":"Job counter-pfwlp does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:09.677Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e5ea9439-349c-407f-80b8-05926b4af325"}
{"level":"error","ts":"2026-07-13T16:00:09.677Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e5ea9439-349c-407f-80b8-05926b4af325","error":"Job counter-pfwlp does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:09.838Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"8508e423-56e7-4071-b9ae-07ffba17564c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:09.838Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"8508e423-56e7-4071-b9ae-07ffba17564c","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:09.838Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"8508e423-56e7-4071-b9ae-07ffba17564c","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:09.839Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"8508e423-56e7-4071-b9ae-07ffba17564c"}
{"level":"error","ts":"2026-07-13T16:00:09.843Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"8508e423-56e7-4071-b9ae-07ffba17564c","JobId":"counter-pfwlp","error":"Job counter-pfwlp does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:09.843Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"8508e423-56e7-4071-b9ae-07ffba17564c"}
{"level":"error","ts":"2026-07-13T16:00:09.843Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"8508e423-56e7-4071-b9ae-07ffba17564c","error":"Job counter-pfwlp does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:10.164Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"358ed5cc-e6b5-40aa-b90e-706dbf1918fb","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:10.164Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"358ed5cc-e6b5-40aa-b90e-706dbf1918fb","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:10.164Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"358ed5cc-e6b5-40aa-b90e-706dbf1918fb","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:10.164Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"358ed5cc-e6b5-40aa-b90e-706dbf1918fb"}
{"level":"error","ts":"2026-07-13T16:00:10.169Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"358ed5cc-e6b5-40aa-b90e-706dbf1918fb","JobId":"counter-pfwlp","error":"Job counter-pfwlp does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:10.169Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"358ed5cc-e6b5-40aa-b90e-706dbf1918fb"}
{"level":"error","ts":"2026-07-13T16:00:10.169Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"358ed5cc-e6b5-40aa-b90e-706dbf1918fb","error":"Job counter-pfwlp does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:10.524Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"2ce0aea2-09b4-48c3-a47f-6ccb4b56b6e1","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:10.524Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"2ce0aea2-09b4-48c3-a47f-6ccb4b56b6e1","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:10.524Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"2ce0aea2-09b4-48c3-a47f-6ccb4b56b6e1","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:10.524Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"2ce0aea2-09b4-48c3-a47f-6ccb4b56b6e1"}
{"level":"error","ts":"2026-07-13T16:00:10.529Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"2ce0aea2-09b4-48c3-a47f-6ccb4b56b6e1","JobId":"counter-pfwlp","error":"Job counter-pfwlp does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:10.529Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"2ce0aea2-09b4-48c3-a47f-6ccb4b56b6e1"}
{"level":"error","ts":"2026-07-13T16:00:10.529Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"2ce0aea2-09b4-48c3-a47f-6ccb4b56b6e1","error":"Job counter-pfwlp does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:10.809Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"2c1f4369-c517-42f0-afc4-8e8712ec3ddb","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:10.809Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"2c1f4369-c517-42f0-afc4-8e8712ec3ddb","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:10.810Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"2c1f4369-c517-42f0-afc4-8e8712ec3ddb","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:10.810Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"2c1f4369-c517-42f0-afc4-8e8712ec3ddb"}
{"level":"error","ts":"2026-07-13T16:00:10.814Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"2c1f4369-c517-42f0-afc4-8e8712ec3ddb","JobId":"counter-pfwlp","error":"Job counter-pfwlp does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:10.814Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"2c1f4369-c517-42f0-afc4-8e8712ec3ddb"}
{"level":"error","ts":"2026-07-13T16:00:10.814Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"2c1f4369-c517-42f0-afc4-8e8712ec3ddb","error":"Job counter-pfwlp does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:11.490Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9b9935d8-057b-49b8-a586-b56ce3a0e646"}
{"level":"info","ts":"2026-07-13T16:00:11.490Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9b9935d8-057b-49b8-a586-b56ce3a0e646"}
{"level":"info","ts":"2026-07-13T16:00:11.490Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9b9935d8-057b-49b8-a586-b56ce3a0e646","1 head service found":"counter-rn54g-head-svc"}
{"level":"info","ts":"2026-07-13T16:00:11.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9b9935d8-057b-49b8-a586-b56ce3a0e646","Found 1 head Pod":"counter-rn54g-head-kw6ml","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:00:11.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9b9935d8-057b-49b8-a586-b56ce3a0e646","head Pod":"counter-rn54g-head-kw6ml","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rn54g-head-kw6ml. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:00:11.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9b9935d8-057b-49b8-a586-b56ce3a0e646","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:00:11.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9b9935d8-057b-49b8-a586-b56ce3a0e646","worker Pod":"counter-rn54g-small-group-worker-rdcgh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-rn54g-small-group-worker-rdcgh. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:00:11.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9b9935d8-057b-49b8-a586-b56ce3a0e646","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:00:11.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9b9935d8-057b-49b8-a586-b56ce3a0e646","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:00:11.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9b9935d8-057b-49b8-a586-b56ce3a0e646","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:00:11.490Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9b9935d8-057b-49b8-a586-b56ce3a0e646","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:00:11.490Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"9b9935d8-057b-49b8-a586-b56ce3a0e646","seconds":300}
{"level":"info","ts":"2026-07-13T16:00:13.376Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"62deb08a-80b1-404f-a10b-71a026d5b6cd","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:13.376Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"62deb08a-80b1-404f-a10b-71a026d5b6cd","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:13.376Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"62deb08a-80b1-404f-a10b-71a026d5b6cd","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:13.376Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"62deb08a-80b1-404f-a10b-71a026d5b6cd"}
{"level":"error","ts":"2026-07-13T16:00:13.381Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"62deb08a-80b1-404f-a10b-71a026d5b6cd","JobId":"counter-pfwlp","error":"Job counter-pfwlp does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:13.381Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"62deb08a-80b1-404f-a10b-71a026d5b6cd"}
{"level":"error","ts":"2026-07-13T16:00:13.381Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"62deb08a-80b1-404f-a10b-71a026d5b6cd","error":"Job counter-pfwlp does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:18.501Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"31d1ac5c-c8da-4786-82e4-ad0ab2a19a31","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:18.501Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"31d1ac5c-c8da-4786-82e4-ad0ab2a19a31","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:18.501Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"31d1ac5c-c8da-4786-82e4-ad0ab2a19a31","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:18.501Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"31d1ac5c-c8da-4786-82e4-ad0ab2a19a31"}
{"level":"info","ts":"2026-07-13T16:00:18.507Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"31d1ac5c-c8da-4786-82e4-ad0ab2a19a31","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","dashboardURL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:00:17Z"},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","dashboardURL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:00:09Z","stateTransitionTimes":{"ready":"2026-07-13T16:00:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:59:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:00:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:00:18.507Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"31d1ac5c-c8da-4786-82e4-ad0ab2a19a31","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T16:00:18.520Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"d7f9bd76-f0ba-40dc-8547-bf3cca0df155","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:18.520Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"d7f9bd76-f0ba-40dc-8547-bf3cca0df155","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:18.520Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"d7f9bd76-f0ba-40dc-8547-bf3cca0df155","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:18.520Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"d7f9bd76-f0ba-40dc-8547-bf3cca0df155"}
{"level":"info","ts":"2026-07-13T16:00:18.522Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"d7f9bd76-f0ba-40dc-8547-bf3cca0df155","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:00:17Z"},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","dashboardURL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:00:09Z","stateTransitionTimes":{"ready":"2026-07-13T16:00:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:59:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:00:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:00:17Z"},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","dashboardURL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:00:09Z","stateTransitionTimes":{"ready":"2026-07-13T16:00:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:59:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:00:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:00:21.520Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"f4e36af7-ae5c-44b6-ba0c-e84544c46401","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:21.520Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"f4e36af7-ae5c-44b6-ba0c-e84544c46401","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:21.520Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"f4e36af7-ae5c-44b6-ba0c-e84544c46401","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:21.521Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"f4e36af7-ae5c-44b6-ba0c-e84544c46401"}
{"level":"info","ts":"2026-07-13T16:00:21.523Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"f4e36af7-ae5c-44b6-ba0c-e84544c46401","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:00:17Z"},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","dashboardURL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:00:09Z","stateTransitionTimes":{"ready":"2026-07-13T16:00:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:59:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:00:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:00:17Z","endTime":"2026-07-13T16:00:21Z"},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","dashboardURL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:00:09Z","stateTransitionTimes":{"ready":"2026-07-13T16:00:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:59:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:00:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:00:21.524Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"f4e36af7-ae5c-44b6-ba0c-e84544c46401","old JobStatus":"PENDING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T16:00:21.537Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"92549f4d-406d-452b-afc6-7d94d2c87856","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:21.537Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"92549f4d-406d-452b-afc6-7d94d2c87856","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:21.537Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"92549f4d-406d-452b-afc6-7d94d2c87856","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:21.537Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"92549f4d-406d-452b-afc6-7d94d2c87856"}
{"level":"info","ts":"2026-07-13T16:00:21.539Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"92549f4d-406d-452b-afc6-7d94d2c87856","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:00:17Z","endTime":"2026-07-13T16:00:21Z"},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","dashboardURL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:00:09Z","stateTransitionTimes":{"ready":"2026-07-13T16:00:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:59:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:00:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:00:17Z","endTime":"2026-07-13T16:00:21Z"},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","dashboardURL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:00:09Z","stateTransitionTimes":{"ready":"2026-07-13T16:00:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:59:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:00:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:00:24.538Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e2733389-b9b1-407b-80e2-bcc0ef5a8007","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:24.538Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e2733389-b9b1-407b-80e2-bcc0ef5a8007","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:24.538Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e2733389-b9b1-407b-80e2-bcc0ef5a8007","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:24.538Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e2733389-b9b1-407b-80e2-bcc0ef5a8007"}
{"level":"info","ts":"2026-07-13T16:00:24.541Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"e2733389-b9b1-407b-80e2-bcc0ef5a8007","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:00:17Z","endTime":"2026-07-13T16:00:21Z"},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","dashboardURL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:00:09Z","stateTransitionTimes":{"ready":"2026-07-13T16:00:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:59:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:00:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:00:17Z","endTime":"2026-07-13T16:00:21Z"},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","dashboardURL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:00:09Z","stateTransitionTimes":{"ready":"2026-07-13T16:00:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:59:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:00:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:00:27.541Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"bf2149cd-48d5-464e-bee0-a12e9cc886b5","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:27.541Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"bf2149cd-48d5-464e-bee0-a12e9cc886b5","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:27.541Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"bf2149cd-48d5-464e-bee0-a12e9cc886b5","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:27.541Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"bf2149cd-48d5-464e-bee0-a12e9cc886b5"}
{"level":"info","ts":"2026-07-13T16:00:27.544Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"bf2149cd-48d5-464e-bee0-a12e9cc886b5","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:00:17Z","endTime":"2026-07-13T16:00:21Z"},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","dashboardURL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:00:09Z","stateTransitionTimes":{"ready":"2026-07-13T16:00:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:59:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:00:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:00:17Z","endTime":"2026-07-13T16:00:21Z"},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","dashboardURL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:00:09Z","stateTransitionTimes":{"ready":"2026-07-13T16:00:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:59:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:00:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:00:28.311Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"cb5e1fd3-fe34-4349-8673-073f2017fa83","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:28.311Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"cb5e1fd3-fe34-4349-8673-073f2017fa83","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:28.311Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"cb5e1fd3-fe34-4349-8673-073f2017fa83","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:28.311Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"cb5e1fd3-fe34-4349-8673-073f2017fa83"}
{"level":"info","ts":"2026-07-13T16:00:28.315Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"cb5e1fd3-fe34-4349-8673-073f2017fa83","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:00:17Z","endTime":"2026-07-13T16:00:21Z"},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","dashboardURL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:00:09Z","stateTransitionTimes":{"ready":"2026-07-13T16:00:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:59:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:00:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:00:17Z","endTime":"2026-07-13T16:00:21Z"},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","dashboardURL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:00:09Z","stateTransitionTimes":{"ready":"2026-07-13T16:00:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:59:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:00:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:00:29.323Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"ec963b49-b129-4655-997f-dcfd81981eaa","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:29.323Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"ec963b49-b129-4655-997f-dcfd81981eaa","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:29.323Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"ec963b49-b129-4655-997f-dcfd81981eaa","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:29.323Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"ec963b49-b129-4655-997f-dcfd81981eaa"}
{"level":"info","ts":"2026-07-13T16:00:29.326Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"ec963b49-b129-4655-997f-dcfd81981eaa","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:00:17Z","endTime":"2026-07-13T16:00:21Z"},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","dashboardURL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:00:09Z","stateTransitionTimes":{"ready":"2026-07-13T16:00:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:59:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:00:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:00:17Z","endTime":"2026-07-13T16:00:21Z"},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","dashboardURL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:00:09Z","stateTransitionTimes":{"ready":"2026-07-13T16:00:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:59:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:00:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:00:29.345Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"4884e5c6-bf88-4ed8-81df-7b486ce39a8a","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:29.345Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"4884e5c6-bf88-4ed8-81df-7b486ce39a8a","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:29.345Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"4884e5c6-bf88-4ed8-81df-7b486ce39a8a","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:29.345Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"4884e5c6-bf88-4ed8-81df-7b486ce39a8a"}
{"level":"info","ts":"2026-07-13T16:00:29.347Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"4884e5c6-bf88-4ed8-81df-7b486ce39a8a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:00:17Z","endTime":"2026-07-13T16:00:21Z"},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","dashboardURL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T15:59:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:00:09Z","stateTransitionTimes":{"ready":"2026-07-13T16:00:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:59:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:00:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:00:17Z","endTime":"2026-07-13T16:00:21Z"},"jobId":"counter-pfwlp","rayClusterName":"counter-rn54g","dashboardURL":"counter-rn54g-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-07-13T15:59:35Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:00:09Z","stateTransitionTimes":{"ready":"2026-07-13T16:00:09Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"counter-rn54g-head-kw6ml","serviceName":"counter-rn54g-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:59:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:00:09Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:59:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:59:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:00:29.347Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"4884e5c6-bf88-4ed8-81df-7b486ce39a8a","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-07-13T16:00:29.366Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"2b356552-c19d-4493-a65f-e9e2c629a2d2","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:29.366Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"2b356552-c19d-4493-a65f-e9e2c629a2d2","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 16:00:29 +0000 UTC","Now":"2026-07-13T16:00:29.366Z","ShutdownTime":"2026-07-13T16:00:29.000Z"}
{"level":"info","ts":"2026-07-13T16:00:29.366Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"2b356552-c19d-4493-a65f-e9e2c629a2d2","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T16:00:29.377Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"2b356552-c19d-4493-a65f-e9e2c629a2d2","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:29.378Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"2b356552-c19d-4493-a65f-e9e2c629a2d2","isClusterDeleted":false}
{"level":"info","ts":"2026-07-13T16:00:29.378Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"2b356552-c19d-4493-a65f-e9e2c629a2d2","RayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:29.379Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"00154cf3-ea4f-44b2-9af5-dfc0dda08d90","namespacedName":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:29.380Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"1fcb51e7-1910-4896-9734-a06d6dd5b4df","rayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:29.380Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"92e85659-2d92-4199-b202-72ff036e6075"}
{"level":"info","ts":"2026-07-13T16:00:29.380Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"00154cf3-ea4f-44b2-9af5-dfc0dda08d90","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:29.380Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"00154cf3-ea4f-44b2-9af5-dfc0dda08d90","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:29.380Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"reconcileID":"e8e8f0c0-1e41-4dd1-96fc-c90432f2f857"}
{"level":"info","ts":"2026-07-13T16:00:29.380Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"af39e3ee-e653-4645-9099-b02820f89c18","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:29.380Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"00154cf3-ea4f-44b2-9af5-dfc0dda08d90","namespace":"test-ns-x5qvg","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-13T16:00:29.380Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"af39e3ee-e653-4645-9099-b02820f89c18","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 16:00:29 +0000 UTC","Now":"2026-07-13T16:00:29.380Z","ShutdownTime":"2026-07-13T16:00:29.000Z"}
{"level":"info","ts":"2026-07-13T16:00:29.380Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"af39e3ee-e653-4645-9099-b02820f89c18","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T16:00:29.380Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"af39e3ee-e653-4645-9099-b02820f89c18","RayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:29.380Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"af39e3ee-e653-4645-9099-b02820f89c18","isClusterDeleted":false}
{"level":"info","ts":"2026-07-13T16:00:29.380Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"af39e3ee-e653-4645-9099-b02820f89c18","RayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:29.380Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"1fcb51e7-1910-4896-9734-a06d6dd5b4df","pod":"counter-rn54g-head-kw6ml","ip":"10.133.0.28"}
{"level":"info","ts":"2026-07-13T16:00:29.380Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"1fcb51e7-1910-4896-9734-a06d6dd5b4df","pod":"counter-rn54g-small-group-worker-rdcgh","ip":"10.134.0.31"}
{"level":"info","ts":"2026-07-13T16:00:29.380Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"1fcb51e7-1910-4896-9734-a06d6dd5b4df","rayCluster":"counter-rn54g","podIPs":["10.133.0.28","10.134.0.31"]}
{"level":"info","ts":"2026-07-13T16:00:29.386Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"00154cf3-ea4f-44b2-9af5-dfc0dda08d90","name":"kuberay-gateway-access","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:00:29.393Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"00154cf3-ea4f-44b2-9af5-dfc0dda08d90","name":"test-ns-x5qvg-counter-rn54g","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-13T16:00:29.400Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"00154cf3-ea4f-44b2-9af5-dfc0dda08d90","configMap":"kube-rbac-proxy-config-counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:29.408Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"00154cf3-ea4f-44b2-9af5-dfc0dda08d90","serviceAccount":"counter-rn54g-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:00:29.416Z","logger":"raycluster-resource","msg":"default","name":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:29.417Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-rn54g","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:00:29.432Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"00154cf3-ea4f-44b2-9af5-dfc0dda08d90","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:29.432Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"501eda06-1faa-43b8-a88f-69e31d7797c1","namespacedName":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:29.432Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"501eda06-1faa-43b8-a88f-69e31d7797c1","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:29.432Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"501eda06-1faa-43b8-a88f-69e31d7797c1","cluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:29.432Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"501eda06-1faa-43b8-a88f-69e31d7797c1","namespace":"test-ns-x5qvg","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-13T16:00:29.432Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"501eda06-1faa-43b8-a88f-69e31d7797c1","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-13T16:00:29.433Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"9f3cabc3-c8b1-480f-bbe5-7fb0dfba0024","namespace":"test-ns-x5qvg","clusterName":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:29.433Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"c32284e8-6d39-49e8-abe4-6eb44ce2e0f0"}
{"level":"info","ts":"2026-07-13T16:00:29.433Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"753011be-736f-4657-b39c-57ee00743b6e","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:29.433Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"753011be-736f-4657-b39c-57ee00743b6e","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 16:00:29 +0000 UTC","Now":"2026-07-13T16:00:29.433Z","ShutdownTime":"2026-07-13T16:00:29.000Z"}
{"level":"info","ts":"2026-07-13T16:00:29.433Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"753011be-736f-4657-b39c-57ee00743b6e","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T16:00:29.433Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"753011be-736f-4657-b39c-57ee00743b6e","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:29.433Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"753011be-736f-4657-b39c-57ee00743b6e","isClusterDeleted":true}
{"level":"info","ts":"2026-07-13T16:00:29.433Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"753011be-736f-4657-b39c-57ee00743b6e","RayCluster":"counter-rn54g"}
{"level":"error","ts":"2026-07-13T16:00:29.443Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"501eda06-1faa-43b8-a88f-69e31d7797c1","error":"rayclusters.ray.io \"counter-rn54g\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:29.443Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"501eda06-1faa-43b8-a88f-69e31d7797c1"}
{"level":"error","ts":"2026-07-13T16:00:29.443Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"501eda06-1faa-43b8-a88f-69e31d7797c1","error":"rayclusters.ray.io \"counter-rn54g\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:29.443Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"da3aed2d-04e8-4cea-8d90-5b6d724bb8d3","namespacedName":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:29.443Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"da3aed2d-04e8-4cea-8d90-5b6d724bb8d3"}
{"level":"info","ts":"2026-07-13T16:00:29.443Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"da3aed2d-04e8-4cea-8d90-5b6d724bb8d3","namespace":"test-ns-x5qvg","authClusters":0}
{"level":"info","ts":"2026-07-13T16:00:29.443Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"da3aed2d-04e8-4cea-8d90-5b6d724bb8d3","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:00:29.449Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"e582338b-73a8-41cf-91bf-9e94d887435b","namespacedName":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:29.449Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"e582338b-73a8-41cf-91bf-9e94d887435b"}
{"level":"info","ts":"2026-07-13T16:00:29.449Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"e582338b-73a8-41cf-91bf-9e94d887435b","namespace":"test-ns-x5qvg","authClusters":0}
{"level":"info","ts":"2026-07-13T16:00:29.449Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"e582338b-73a8-41cf-91bf-9e94d887435b","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:00:29.455Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"85aba0b0-2264-4b45-917d-e8f905c4d6d2"}
{"level":"info","ts":"2026-07-13T16:00:29.457Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"b2df71aa-4d27-4b3d-8db5-b91deed3556f"}
{"level":"info","ts":"2026-07-13T16:00:29.470Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"cf50e907-e600-43fb-97c2-d0c4663abbec","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:29.470Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"cf50e907-e600-43fb-97c2-d0c4663abbec","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 16:00:29 +0000 UTC","Now":"2026-07-13T16:00:29.470Z","ShutdownTime":"2026-07-13T16:00:29.000Z"}
{"level":"info","ts":"2026-07-13T16:00:29.470Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"cf50e907-e600-43fb-97c2-d0c4663abbec","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T16:00:29.470Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"cf50e907-e600-43fb-97c2-d0c4663abbec","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:29.470Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"cf50e907-e600-43fb-97c2-d0c4663abbec","isClusterDeleted":true}
{"level":"info","ts":"2026-07-13T16:00:29.470Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"cf50e907-e600-43fb-97c2-d0c4663abbec","RayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:29.479Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"94c1fff3-fcaa-4833-b839-e9130cc4eaec","namespacedName":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:29.479Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"94c1fff3-fcaa-4833-b839-e9130cc4eaec"}
{"level":"info","ts":"2026-07-13T16:00:29.479Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"94c1fff3-fcaa-4833-b839-e9130cc4eaec","namespace":"test-ns-x5qvg","authClusters":0}
{"level":"info","ts":"2026-07-13T16:00:29.479Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"94c1fff3-fcaa-4833-b839-e9130cc4eaec","namespace":"test-ns-x5qvg"}
{"level":"error","ts":"2026-07-13T16:00:29.485Z","msg":"Failed to delete CA certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"9f3cabc3-c8b1-480f-bbe5-7fb0dfba0024","certificate":"ray-ca-certificate-counter-rn54g","error":"certificates.cert-manager.io \"ray-ca-certificate-counter-rn54g\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:275\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:29.621Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"9f3cabc3-c8b1-480f-bbe5-7fb0dfba0024","errorCount":1}
{"level":"info","ts":"2026-07-13T16:00:29.621Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"9f3cabc3-c8b1-480f-bbe5-7fb0dfba0024"}
{"level":"error","ts":"2026-07-13T16:00:29.621Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"9f3cabc3-c8b1-480f-bbe5-7fb0dfba0024","error":"certificates.cert-manager.io \"ray-ca-certificate-counter-rn54g\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:29.626Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"d4921193-cbe7-4199-aed8-55ddc02a3ef7","namespace":"test-ns-x5qvg","clusterName":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:29.626Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"counter-rn54g","reconcileID":"d4921193-cbe7-4199-aed8-55ddc02a3ef7","namespace":"test-ns-x5qvg","clusterName":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:30.545Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"eb65678a-1373-4f22-b70a-d4fb29b28049","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:30.545Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"eb65678a-1373-4f22-b70a-d4fb29b28049","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 16:00:29 +0000 UTC","Now":"2026-07-13T16:00:30.545Z","ShutdownTime":"2026-07-13T16:00:29.000Z"}
{"level":"info","ts":"2026-07-13T16:00:30.545Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"eb65678a-1373-4f22-b70a-d4fb29b28049","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T16:00:30.545Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"eb65678a-1373-4f22-b70a-d4fb29b28049","RayCluster":{"name":"counter-rn54g","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:30.545Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"eb65678a-1373-4f22-b70a-d4fb29b28049","isClusterDeleted":true}
{"level":"info","ts":"2026-07-13T16:00:30.545Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"eb65678a-1373-4f22-b70a-d4fb29b28049","RayCluster":"counter-rn54g"}
{"level":"info","ts":"2026-07-13T16:00:34.942Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a47d200d-483a-445d-8a73-08f811e3cbaf","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:00:34.943Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a47d200d-483a-445d-8a73-08f811e3cbaf","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T16:00:34.943Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a47d200d-483a-445d-8a73-08f811e3cbaf","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T16:00:34.943Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a47d200d-483a-445d-8a73-08f811e3cbaf","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T16:00:59.483Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"b1c3d60f-7036-4d4c-9390-41a003be05ed","DeletionTimestamp":"2026-07-13 16:00:59 +0000 UTC"}
{"level":"info","ts":"2026-07-13T16:00:59.483Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"b1c3d60f-7036-4d4c-9390-41a003be05ed","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T16:00:59.508Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d34ef6e2-1898-4f16-87fd-4c753b02dfcc","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:59.508Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d34ef6e2-1898-4f16-87fd-4c753b02dfcc","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T16:00:59.520Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d34ef6e2-1898-4f16-87fd-4c753b02dfcc"}
{"level":"info","ts":"2026-07-13T16:00:59.520Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d34ef6e2-1898-4f16-87fd-4c753b02dfcc","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-13T16:00:59.520Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d34ef6e2-1898-4f16-87fd-4c753b02dfcc","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T16:00:59.520Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d34ef6e2-1898-4f16-87fd-4c753b02dfcc","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T16:00:59.533Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"6566e167-2c99-4335-8858-2650f299b8e5"}
{"level":"info","ts":"2026-07-13T16:00:59.533Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"76fc5918-6cff-4975-97bd-aec59940371f","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:59.533Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"76fc5918-6cff-4975-97bd-aec59940371f"}
{"level":"info","ts":"2026-07-13T16:00:59.533Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"76fc5918-6cff-4975-97bd-aec59940371f","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-13T16:00:59.533Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"76fc5918-6cff-4975-97bd-aec59940371f","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-fwc7g","rayClusterName":"fail-mwjqw","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T16:00:59.533Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"76fc5918-6cff-4975-97bd-aec59940371f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T16:00:59.542Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"76fc5918-6cff-4975-97bd-aec59940371f","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-13T16:00:59.542Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"76fc5918-6cff-4975-97bd-aec59940371f"}
{"level":"error","ts":"2026-07-13T16:00:59.542Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"76fc5918-6cff-4975-97bd-aec59940371f","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.542Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"c02534e0-614e-43e7-8f34-bf6ec3fc9de9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:59.542Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"c02534e0-614e-43e7-8f34-bf6ec3fc9de9","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.542Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"c02534e0-614e-43e7-8f34-bf6ec3fc9de9","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:59.542Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"c02534e0-614e-43e7-8f34-bf6ec3fc9de9","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:59.549Z","logger":"raycluster-resource","msg":"default","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.549Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-76mdf","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:00:59.566Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"dc496747-e508-4247-89d9-0143448edc0f","namespacedName":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:59.566Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"d9f9631a-a82c-469d-afcf-a36c8f1db4a7","rayCluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.566Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"dc496747-e508-4247-89d9-0143448edc0f","mode":"IntegratedOAuth","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.566Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"dc496747-e508-4247-89d9-0143448edc0f","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.566Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"d9f9631a-a82c-469d-afcf-a36c8f1db4a7","rayCluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.566Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"dc496747-e508-4247-89d9-0143448edc0f","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.566Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"2f159ac4-fb6a-42ae-a0e1-e37e8c99edce"}
{"level":"info","ts":"2026-07-13T16:00:59.566Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"2f159ac4-fb6a-42ae-a0e1-e37e8c99edce"}
{"level":"info","ts":"2026-07-13T16:00:59.566Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"5362ebe2-66be-4da6-94a6-21655cbd3922","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.566Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"c02534e0-614e-43e7-8f34-bf6ec3fc9de9","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:59.567Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"c02534e0-614e-43e7-8f34-bf6ec3fc9de9"}
{"level":"info","ts":"2026-07-13T16:00:59.567Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"c02534e0-614e-43e7-8f34-bf6ec3fc9de9","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:00:59.567Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d82c5615-d549-496b-a236-d7a24f909470","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:59.567Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d82c5615-d549-496b-a236-d7a24f909470","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.567Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d82c5615-d549-496b-a236-d7a24f909470","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:59.567Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d82c5615-d549-496b-a236-d7a24f909470"}
{"level":"info","ts":"2026-07-13T16:00:59.567Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d82c5615-d549-496b-a236-d7a24f909470","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:00:59.573Z","logger":"raycluster-resource","msg":"default","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.573Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-76mdf","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:00:59.575Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"2f159ac4-fb6a-42ae-a0e1-e37e8c99edce","name":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:00:59.575Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"2f159ac4-fb6a-42ae-a0e1-e37e8c99edce","error":"head certificate secret ray-head-secret-fail-76mdf not found"}
{"level":"error","ts":"2026-07-13T16:00:59.575Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"2f159ac4-fb6a-42ae-a0e1-e37e8c99edce","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.575Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"2f159ac4-fb6a-42ae-a0e1-e37e8c99edce","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found"}
{"level":"info","ts":"2026-07-13T16:00:59.575Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"2f159ac4-fb6a-42ae-a0e1-e37e8c99edce","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:00:59.575Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"2f159ac4-fb6a-42ae-a0e1-e37e8c99edce","serviceSelectors":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/identifier":"fail-76mdf-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:00:59.575Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"2f159ac4-fb6a-42ae-a0e1-e37e8c99edce","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:00:59.575Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"2f159ac4-fb6a-42ae-a0e1-e37e8c99edce","error":"unable to find head service. cluster name fail-76mdf, filter labels [test-ns-x5qvg map[ray.io/cluster:fail-76mdf ray.io/identifier:fail-76mdf-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-07-13T16:00:59.575Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"2f159ac4-fb6a-42ae-a0e1-e37e8c99edce"}
{"level":"error","ts":"2026-07-13T16:00:59.575Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"2f159ac4-fb6a-42ae-a0e1-e37e8c99edce","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.575Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"efca0a05-a1a9-422c-8d22-3d65533f427e"}
{"level":"info","ts":"2026-07-13T16:00:59.575Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"efca0a05-a1a9-422c-8d22-3d65533f427e"}
{"level":"info","ts":"2026-07-13T16:00:59.575Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"efca0a05-a1a9-422c-8d22-3d65533f427e","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:00:59.575Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"efca0a05-a1a9-422c-8d22-3d65533f427e","error":"head certificate secret ray-head-secret-fail-76mdf not found"}
{"level":"error","ts":"2026-07-13T16:00:59.575Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"efca0a05-a1a9-422c-8d22-3d65533f427e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.575Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"efca0a05-a1a9-422c-8d22-3d65533f427e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found"}
{"level":"info","ts":"2026-07-13T16:00:59.575Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"efca0a05-a1a9-422c-8d22-3d65533f427e","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:00:59.575Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"efca0a05-a1a9-422c-8d22-3d65533f427e","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:00:59.575Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"efca0a05-a1a9-422c-8d22-3d65533f427e","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:00:59.575Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"efca0a05-a1a9-422c-8d22-3d65533f427e","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T16:00:59.575Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"efca0a05-a1a9-422c-8d22-3d65533f427e","name":"fail-76mdf","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:00:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:00:59.578Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"d9f9631a-a82c-469d-afcf-a36c8f1db4a7","rayCluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.578Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"d9f9631a-a82c-469d-afcf-a36c8f1db4a7","rayCluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.585Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"08cdde71-24c4-4eca-9ab8-9d975de63663","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:59.585Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"08cdde71-24c4-4eca-9ab8-9d975de63663","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.585Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"08cdde71-24c4-4eca-9ab8-9d975de63663","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:59.585Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"08cdde71-24c4-4eca-9ab8-9d975de63663"}
{"level":"info","ts":"2026-07-13T16:00:59.621Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"08cdde71-24c4-4eca-9ab8-9d975de63663","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:00:59.621Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"5362ebe2-66be-4da6-94a6-21655cbd3922","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.621Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"a9a995ed-2628-48f2-b5dd-c9d6ca0ea8e0","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.621Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"efca0a05-a1a9-422c-8d22-3d65533f427e","name":"fail-76mdf","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-76mdf\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-x5qvg","name":"fail-76mdf"}}
{"level":"info","ts":"2026-07-13T16:00:59.621Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"efca0a05-a1a9-422c-8d22-3d65533f427e"}
{"level":"error","ts":"2026-07-13T16:00:59.621Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"efca0a05-a1a9-422c-8d22-3d65533f427e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.621Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"d9f9631a-a82c-469d-afcf-a36c8f1db4a7","rayCluster":"fail-76mdf","secret":"ca-secret"}
{"level":"info","ts":"2026-07-13T16:00:59.621Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"d9f9631a-a82c-469d-afcf-a36c8f1db4a7","rayCluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.621Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"df6a0a08-20c4-475b-a804-ce4573d6ea9e"}
{"level":"info","ts":"2026-07-13T16:00:59.621Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"df6a0a08-20c4-475b-a804-ce4573d6ea9e"}
{"level":"info","ts":"2026-07-13T16:00:59.621Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"df6a0a08-20c4-475b-a804-ce4573d6ea9e","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:00:59.621Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"df6a0a08-20c4-475b-a804-ce4573d6ea9e","error":"head certificate secret ray-head-secret-fail-76mdf not found"}
{"level":"error","ts":"2026-07-13T16:00:59.621Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"df6a0a08-20c4-475b-a804-ce4573d6ea9e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.621Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"df6a0a08-20c4-475b-a804-ce4573d6ea9e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found"}
{"level":"info","ts":"2026-07-13T16:00:59.621Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"df6a0a08-20c4-475b-a804-ce4573d6ea9e","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:00:59.621Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"df6a0a08-20c4-475b-a804-ce4573d6ea9e","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:00:59.621Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"df6a0a08-20c4-475b-a804-ce4573d6ea9e","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:00:59.621Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"df6a0a08-20c4-475b-a804-ce4573d6ea9e","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T16:00:59.621Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"df6a0a08-20c4-475b-a804-ce4573d6ea9e","name":"fail-76mdf","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:00:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:00:59.627Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"dc496747-e508-4247-89d9-0143448edc0f","name":"fail-76mdf-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:00:59.636Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"d9f9631a-a82c-469d-afcf-a36c8f1db4a7","rayCluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.640Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"dc496747-e508-4247-89d9-0143448edc0f","name":"kuberay-gateway-access","namespace":"test-ns-x5qvg","operation":"created"}
{"level":"info","ts":"2026-07-13T16:00:59.640Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"b37133b4-e27c-4e3b-a144-60109f05b7cc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:59.640Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"b37133b4-e27c-4e3b-a144-60109f05b7cc","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.640Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"b37133b4-e27c-4e3b-a144-60109f05b7cc","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:59.640Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"b37133b4-e27c-4e3b-a144-60109f05b7cc"}
{"level":"info","ts":"2026-07-13T16:00:59.640Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"b37133b4-e27c-4e3b-a144-60109f05b7cc","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:00:59.641Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"df6a0a08-20c4-475b-a804-ce4573d6ea9e"}
{"level":"error","ts":"2026-07-13T16:00:59.641Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"df6a0a08-20c4-475b-a804-ce4573d6ea9e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.641Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"814c894d-f4e0-43f9-bd17-25f8334016e4"}
{"level":"info","ts":"2026-07-13T16:00:59.641Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"814c894d-f4e0-43f9-bd17-25f8334016e4"}
{"level":"info","ts":"2026-07-13T16:00:59.641Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"814c894d-f4e0-43f9-bd17-25f8334016e4","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:00:59.641Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"814c894d-f4e0-43f9-bd17-25f8334016e4","error":"head certificate secret ray-head-secret-fail-76mdf not found"}
{"level":"error","ts":"2026-07-13T16:00:59.641Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"814c894d-f4e0-43f9-bd17-25f8334016e4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.641Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"814c894d-f4e0-43f9-bd17-25f8334016e4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found"}
{"level":"info","ts":"2026-07-13T16:00:59.641Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"814c894d-f4e0-43f9-bd17-25f8334016e4","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:00:59.641Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"814c894d-f4e0-43f9-bd17-25f8334016e4","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:00:59.641Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"814c894d-f4e0-43f9-bd17-25f8334016e4","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:00:59.641Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"814c894d-f4e0-43f9-bd17-25f8334016e4"}
{"level":"error","ts":"2026-07-13T16:00:59.641Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"814c894d-f4e0-43f9-bd17-25f8334016e4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.645Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"a9a995ed-2628-48f2-b5dd-c9d6ca0ea8e0","name":"fail-76mdf-head"}
{"level":"info","ts":"2026-07-13T16:00:59.649Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"d9f9631a-a82c-469d-afcf-a36c8f1db4a7","rayCluster":"fail-76mdf","podIPs":[]}
{"level":"info","ts":"2026-07-13T16:00:59.650Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"dc496747-e508-4247-89d9-0143448edc0f","name":"test-ns-x5qvg-fail-76mdf","operation":"created"}
{"level":"info","ts":"2026-07-13T16:00:59.656Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"dc496747-e508-4247-89d9-0143448edc0f","name":"kube-rbac-proxy-config-fail-76mdf","operation":"created"}
{"level":"info","ts":"2026-07-13T16:00:59.659Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"a9a995ed-2628-48f2-b5dd-c9d6ca0ea8e0","name":"fail-76mdf-workers"}
{"level":"info","ts":"2026-07-13T16:00:59.659Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"a9a995ed-2628-48f2-b5dd-c9d6ca0ea8e0","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.659Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"3642b652-5d26-4004-b32e-b91674a8a36c","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.721Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"e3ddaac2-946a-4728-b5cc-abb00a934796"}
{"level":"info","ts":"2026-07-13T16:00:59.721Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"e3ddaac2-946a-4728-b5cc-abb00a934796"}
{"level":"info","ts":"2026-07-13T16:00:59.721Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"e3ddaac2-946a-4728-b5cc-abb00a934796","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:00:59.721Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"e3ddaac2-946a-4728-b5cc-abb00a934796","error":"head certificate secret ray-head-secret-fail-76mdf not found"}
{"level":"error","ts":"2026-07-13T16:00:59.721Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"e3ddaac2-946a-4728-b5cc-abb00a934796","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.721Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"e3ddaac2-946a-4728-b5cc-abb00a934796","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found"}
{"level":"info","ts":"2026-07-13T16:00:59.721Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"e3ddaac2-946a-4728-b5cc-abb00a934796","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:00:59.721Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"e3ddaac2-946a-4728-b5cc-abb00a934796","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:00:59.721Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"e3ddaac2-946a-4728-b5cc-abb00a934796","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:00:59.721Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"e3ddaac2-946a-4728-b5cc-abb00a934796"}
{"level":"error","ts":"2026-07-13T16:00:59.721Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"e3ddaac2-946a-4728-b5cc-abb00a934796","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.721Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"d9f9631a-a82c-469d-afcf-a36c8f1db4a7","rayCluster":"fail-76mdf","podIPs":[]}
{"level":"error","ts":"2026-07-13T16:00:59.721Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"d9f9631a-a82c-469d-afcf-a36c8f1db4a7","error":"Certificate.cert-manager.io \"ray-worker-cert-fail-76mdf\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.722Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"d9f9631a-a82c-469d-afcf-a36c8f1db4a7"}
{"level":"error","ts":"2026-07-13T16:00:59.722Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"d9f9631a-a82c-469d-afcf-a36c8f1db4a7","error":"Certificate.cert-manager.io \"ray-worker-cert-fail-76mdf\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.722Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"6e59036c-e4f5-497e-99c1-6db50b272e30","rayCluster":"fail-76mdf"}
{"level":"error","ts":"2026-07-13T16:00:59.722Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"dc496747-e508-4247-89d9-0143448edc0f","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-76mdf\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.722Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"dc496747-e508-4247-89d9-0143448edc0f"}
{"level":"error","ts":"2026-07-13T16:00:59.722Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"dc496747-e508-4247-89d9-0143448edc0f","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-76mdf\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.722Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"a2e56a66-6741-4f14-bfd5-faba06f3c5d4","namespacedName":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:59.722Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"a2e56a66-6741-4f14-bfd5-faba06f3c5d4","mode":"IntegratedOAuth","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.722Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"a2e56a66-6741-4f14-bfd5-faba06f3c5d4","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.726Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"3642b652-5d26-4004-b32e-b91674a8a36c","name":"fail-76mdf-head"}
{"level":"info","ts":"2026-07-13T16:00:59.735Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"a2e56a66-6741-4f14-bfd5-faba06f3c5d4","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.735Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"564e9ba7-5825-477d-bfdb-2ccd47be4475","namespacedName":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:59.735Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"564e9ba7-5825-477d-bfdb-2ccd47be4475","mode":"IntegratedOAuth","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.735Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"564e9ba7-5825-477d-bfdb-2ccd47be4475","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.735Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"6ab5f7bd-732b-46e6-b385-e461d195565f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:00:59.735Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"6ab5f7bd-732b-46e6-b385-e461d195565f","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.735Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"6ab5f7bd-732b-46e6-b385-e461d195565f","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:59.735Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"6ab5f7bd-732b-46e6-b385-e461d195565f"}
{"level":"info","ts":"2026-07-13T16:00:59.736Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"6ab5f7bd-732b-46e6-b385-e461d195565f","RayCluster":"fail-76mdf","State":""}
{"level":"error","ts":"2026-07-13T16:00:59.736Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"6e59036c-e4f5-497e-99c1-6db50b272e30","error":"certificates.cert-manager.io \"ray-worker-cert-fail-76mdf\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:00:59.736Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"6e59036c-e4f5-497e-99c1-6db50b272e30","error":"certificates.cert-manager.io \"ray-worker-cert-fail-76mdf\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.736Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"6e59036c-e4f5-497e-99c1-6db50b272e30"}
{"level":"error","ts":"2026-07-13T16:00:59.736Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"6e59036c-e4f5-497e-99c1-6db50b272e30","error":"certificates.cert-manager.io \"ray-worker-cert-fail-76mdf\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.736Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"aaed606e-bc99-41bd-aeab-0e12823488bd","rayCluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.736Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"aaed606e-bc99-41bd-aeab-0e12823488bd"}
{"level":"info","ts":"2026-07-13T16:00:59.739Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"3642b652-5d26-4004-b32e-b91674a8a36c","name":"fail-76mdf-workers"}
{"level":"info","ts":"2026-07-13T16:00:59.739Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"3642b652-5d26-4004-b32e-b91674a8a36c","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.739Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"6754ed1b-35c2-4952-b2bd-d610b3c5d974","cluster":"fail-76mdf"}
{"level":"error","ts":"2026-07-13T16:00:59.745Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"564e9ba7-5825-477d-bfdb-2ccd47be4475","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-76mdf\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.745Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"564e9ba7-5825-477d-bfdb-2ccd47be4475"}
{"level":"error","ts":"2026-07-13T16:00:59.745Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"564e9ba7-5825-477d-bfdb-2ccd47be4475","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-76mdf\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.821Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"13fbcc2e-20d1-4aee-9ecd-f213a3ac80c4"}
{"level":"info","ts":"2026-07-13T16:00:59.821Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"13fbcc2e-20d1-4aee-9ecd-f213a3ac80c4"}
{"level":"info","ts":"2026-07-13T16:00:59.821Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"7f714a43-c82c-437c-a782-5171e61f8062","namespacedName":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:00:59.821Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"7f714a43-c82c-437c-a782-5171e61f8062","mode":"IntegratedOAuth","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.821Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"7f714a43-c82c-437c-a782-5171e61f8062","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.821Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"13fbcc2e-20d1-4aee-9ecd-f213a3ac80c4","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:00:59.821Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"13fbcc2e-20d1-4aee-9ecd-f213a3ac80c4","error":"head certificate secret ray-head-secret-fail-76mdf not found"}
{"level":"error","ts":"2026-07-13T16:00:59.821Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"13fbcc2e-20d1-4aee-9ecd-f213a3ac80c4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.821Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"13fbcc2e-20d1-4aee-9ecd-f213a3ac80c4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found"}
{"level":"info","ts":"2026-07-13T16:00:59.821Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"e94bf89c-f5c9-4f84-83fd-bd681d55d824","rayCluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.821Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"e94bf89c-f5c9-4f84-83fd-bd681d55d824"}
{"level":"info","ts":"2026-07-13T16:00:59.821Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"13fbcc2e-20d1-4aee-9ecd-f213a3ac80c4","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:00:59.821Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"13fbcc2e-20d1-4aee-9ecd-f213a3ac80c4","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:00:59.821Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"13fbcc2e-20d1-4aee-9ecd-f213a3ac80c4","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:00:59.821Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"13fbcc2e-20d1-4aee-9ecd-f213a3ac80c4"}
{"level":"error","ts":"2026-07-13T16:00:59.821Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"13fbcc2e-20d1-4aee-9ecd-f213a3ac80c4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.826Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"6754ed1b-35c2-4952-b2bd-d610b3c5d974","name":"fail-76mdf-head"}
{"level":"info","ts":"2026-07-13T16:00:59.835Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"7f714a43-c82c-437c-a782-5171e61f8062","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.921Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"6754ed1b-35c2-4952-b2bd-d610b3c5d974","name":"fail-76mdf-workers"}
{"level":"info","ts":"2026-07-13T16:00:59.921Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"6754ed1b-35c2-4952-b2bd-d610b3c5d974","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:00:59.982Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6ec870c7-c895-4721-ade2-c86e863d0f1d"}
{"level":"info","ts":"2026-07-13T16:00:59.982Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6ec870c7-c895-4721-ade2-c86e863d0f1d"}
{"level":"info","ts":"2026-07-13T16:00:59.982Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6ec870c7-c895-4721-ade2-c86e863d0f1d","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:00:59.982Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6ec870c7-c895-4721-ade2-c86e863d0f1d","error":"head certificate secret ray-head-secret-fail-76mdf not found"}
{"level":"error","ts":"2026-07-13T16:00:59.982Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6ec870c7-c895-4721-ade2-c86e863d0f1d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:00:59.982Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6ec870c7-c895-4721-ade2-c86e863d0f1d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found"}
{"level":"info","ts":"2026-07-13T16:00:59.983Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6ec870c7-c895-4721-ade2-c86e863d0f1d","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:00:59.983Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6ec870c7-c895-4721-ade2-c86e863d0f1d","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:00:59.983Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6ec870c7-c895-4721-ade2-c86e863d0f1d","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:00:59.983Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6ec870c7-c895-4721-ade2-c86e863d0f1d"}
{"level":"error","ts":"2026-07-13T16:00:59.983Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6ec870c7-c895-4721-ade2-c86e863d0f1d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:01:00.303Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6b7d13ec-1f13-41b9-ae81-23287c65ebd3"}
{"level":"info","ts":"2026-07-13T16:01:00.303Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6b7d13ec-1f13-41b9-ae81-23287c65ebd3"}
{"level":"info","ts":"2026-07-13T16:01:00.303Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6b7d13ec-1f13-41b9-ae81-23287c65ebd3","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:01:00.303Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6b7d13ec-1f13-41b9-ae81-23287c65ebd3","error":"head certificate secret ray-head-secret-fail-76mdf not found"}
{"level":"error","ts":"2026-07-13T16:01:00.303Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6b7d13ec-1f13-41b9-ae81-23287c65ebd3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:01:00.303Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6b7d13ec-1f13-41b9-ae81-23287c65ebd3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found"}
{"level":"info","ts":"2026-07-13T16:01:00.303Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6b7d13ec-1f13-41b9-ae81-23287c65ebd3","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:01:00.303Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6b7d13ec-1f13-41b9-ae81-23287c65ebd3","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:01:00.303Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6b7d13ec-1f13-41b9-ae81-23287c65ebd3","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:01:00.303Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6b7d13ec-1f13-41b9-ae81-23287c65ebd3"}
{"level":"error","ts":"2026-07-13T16:01:00.303Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"6b7d13ec-1f13-41b9-ae81-23287c65ebd3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-76mdf not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:01:00.944Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a6163ac3-f3cf-4f5e-a475-40f02fab3222"}
{"level":"info","ts":"2026-07-13T16:01:00.944Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a6163ac3-f3cf-4f5e-a475-40f02fab3222"}
{"level":"info","ts":"2026-07-13T16:01:00.945Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a6163ac3-f3cf-4f5e-a475-40f02fab3222","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:01:00.945Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a6163ac3-f3cf-4f5e-a475-40f02fab3222","error":"worker certificate secret ray-worker-secret-fail-76mdf not found"}
{"level":"error","ts":"2026-07-13T16:01:00.945Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a6163ac3-f3cf-4f5e-a475-40f02fab3222","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-fail-76mdf not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:01:00.945Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a6163ac3-f3cf-4f5e-a475-40f02fab3222","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-fail-76mdf not found"}
{"level":"info","ts":"2026-07-13T16:01:00.945Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a6163ac3-f3cf-4f5e-a475-40f02fab3222","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:01:00.945Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a6163ac3-f3cf-4f5e-a475-40f02fab3222","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:01:00.945Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a6163ac3-f3cf-4f5e-a475-40f02fab3222","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-76mdf","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:01:00.945Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a6163ac3-f3cf-4f5e-a475-40f02fab3222"}
{"level":"error","ts":"2026-07-13T16:01:00.945Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a6163ac3-f3cf-4f5e-a475-40f02fab3222","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-fail-76mdf not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:01:02.225Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40"}
{"level":"info","ts":"2026-07-13T16:01:02.225Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40"}
{"level":"info","ts":"2026-07-13T16:01:02.225Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:01:02.225Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40"}
{"level":"info","ts":"2026-07-13T16:01:02.225Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:01:02.225Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40"}
{"level":"info","ts":"2026-07-13T16:01:02.225Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","mode":"IntegratedOAuth","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:02.225Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","cluster":"fail-76mdf","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:01:02.225Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","cluster":"fail-76mdf","authType":"IntegratedOAuth","serviceAccount":"fail-76mdf-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T16:01:02.225Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-76mdf","ray.io/group":"headgroup","ray.io/identifier":"fail-76mdf-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:01:02.225Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T16:01:02.225Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T16:01:02.225Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T16:01:02.225Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","enabled":true}
{"level":"info","ts":"2026-07-13T16:01:02.248Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","name":"fail-76mdf-head-rbw2b"}
{"level":"info","ts":"2026-07-13T16:01:02.248Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:01:02.248Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:02.248Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:01:02.248Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:02.248Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:01:02.248Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40"}
{"level":"info","ts":"2026-07-13T16:01:02.248Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","nodeType":"worker","rayStartParams":{"address":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:01:02.248Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","rayStartCmd":"ray start  --address=fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:01:02.248Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:01:02.248Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","enabled":true}
{"level":"info","ts":"2026-07-13T16:01:02.270Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","name":"fail-76mdf-small-group-worker-xzxgm"}
{"level":"info","ts":"2026-07-13T16:01:02.270Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","podName":"fail-76mdf-head-rbw2b","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:02.270Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-76mdf-head-svc"},"newHead":{"podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"}}
{"level":"info","ts":"2026-07-13T16:01:02.270Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"242032ec-d325-4290-b485-ae3150bb0c40","name":"fail-76mdf","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:00:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:01:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:01:02.287Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"046aab8f-8d9f-470f-8873-f995dac7166f"}
{"level":"info","ts":"2026-07-13T16:01:02.287Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"046aab8f-8d9f-470f-8873-f995dac7166f"}
{"level":"info","ts":"2026-07-13T16:01:02.287Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"046aab8f-8d9f-470f-8873-f995dac7166f","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:01:02.287Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"1320040c-3795-4e0b-8684-871f6fb8def1","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:02.287Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"046aab8f-8d9f-470f-8873-f995dac7166f","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:01:02.287Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"046aab8f-8d9f-470f-8873-f995dac7166f","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:02.287Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"046aab8f-8d9f-470f-8873-f995dac7166f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:01:02.287Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"046aab8f-8d9f-470f-8873-f995dac7166f","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:02.287Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"046aab8f-8d9f-470f-8873-f995dac7166f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:02.287Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"046aab8f-8d9f-470f-8873-f995dac7166f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:01:02.287Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"046aab8f-8d9f-470f-8873-f995dac7166f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:02.287Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"046aab8f-8d9f-470f-8873-f995dac7166f","podName":"fail-76mdf-head-rbw2b","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:02.287Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"046aab8f-8d9f-470f-8873-f995dac7166f","podName":"fail-76mdf-head-rbw2b","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:02.287Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"046aab8f-8d9f-470f-8873-f995dac7166f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:01:02.287Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"046aab8f-8d9f-470f-8873-f995dac7166f","seconds":300}
{"level":"info","ts":"2026-07-13T16:01:02.287Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cdacc7d9-8ff1-4680-9027-048c4de1d689","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:02.287Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cdacc7d9-8ff1-4680-9027-048c4de1d689","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:02.287Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cdacc7d9-8ff1-4680-9027-048c4de1d689","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:02.288Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cdacc7d9-8ff1-4680-9027-048c4de1d689"}
{"level":"info","ts":"2026-07-13T16:01:02.288Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cdacc7d9-8ff1-4680-9027-048c4de1d689","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:02.296Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"80e40c9c-5105-4e79-a04a-d358fc7bb15e"}
{"level":"info","ts":"2026-07-13T16:01:02.296Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"80e40c9c-5105-4e79-a04a-d358fc7bb15e"}
{"level":"info","ts":"2026-07-13T16:01:02.296Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"80e40c9c-5105-4e79-a04a-d358fc7bb15e","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:01:02.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"80e40c9c-5105-4e79-a04a-d358fc7bb15e","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:01:02.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"80e40c9c-5105-4e79-a04a-d358fc7bb15e","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:02.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"80e40c9c-5105-4e79-a04a-d358fc7bb15e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:01:02.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"80e40c9c-5105-4e79-a04a-d358fc7bb15e","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:02.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"80e40c9c-5105-4e79-a04a-d358fc7bb15e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:02.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"80e40c9c-5105-4e79-a04a-d358fc7bb15e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:01:02.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"80e40c9c-5105-4e79-a04a-d358fc7bb15e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:02.297Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"80e40c9c-5105-4e79-a04a-d358fc7bb15e","podName":"fail-76mdf-head-rbw2b","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:02.297Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"80e40c9c-5105-4e79-a04a-d358fc7bb15e","podName":"fail-76mdf-head-rbw2b","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:02.297Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"80e40c9c-5105-4e79-a04a-d358fc7bb15e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:01:02.297Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"80e40c9c-5105-4e79-a04a-d358fc7bb15e","seconds":300}
{"level":"info","ts":"2026-07-13T16:01:02.311Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c213a3e4-9ab0-476a-819a-133c73a90ae1"}
{"level":"info","ts":"2026-07-13T16:01:02.311Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c213a3e4-9ab0-476a-819a-133c73a90ae1"}
{"level":"info","ts":"2026-07-13T16:01:02.311Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c213a3e4-9ab0-476a-819a-133c73a90ae1","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:01:02.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c213a3e4-9ab0-476a-819a-133c73a90ae1","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:01:02.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c213a3e4-9ab0-476a-819a-133c73a90ae1","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:02.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c213a3e4-9ab0-476a-819a-133c73a90ae1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:01:02.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c213a3e4-9ab0-476a-819a-133c73a90ae1","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:02.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c213a3e4-9ab0-476a-819a-133c73a90ae1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:02.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c213a3e4-9ab0-476a-819a-133c73a90ae1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:01:02.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c213a3e4-9ab0-476a-819a-133c73a90ae1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:02.311Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c213a3e4-9ab0-476a-819a-133c73a90ae1","podName":"fail-76mdf-head-rbw2b","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:02.311Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c213a3e4-9ab0-476a-819a-133c73a90ae1","podName":"fail-76mdf-head-rbw2b","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:02.311Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c213a3e4-9ab0-476a-819a-133c73a90ae1","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T16:01:02.311Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c213a3e4-9ab0-476a-819a-133c73a90ae1","name":"fail-76mdf","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:01:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:01:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:01:02.321Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"dfb63b91-2461-42b6-b229-20746a9507f7","rayCluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:02.321Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"dfb63b91-2461-42b6-b229-20746a9507f7","rayCluster":"fail-76mdf","podIPs":[]}
{"level":"info","ts":"2026-07-13T16:01:02.325Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"1320040c-3795-4e0b-8684-871f6fb8def1","name":"fail-76mdf-head"}
{"level":"info","ts":"2026-07-13T16:01:02.337Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c1ca4018-30a6-4e85-92b5-e0dc500559b7"}
{"level":"info","ts":"2026-07-13T16:01:02.337Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c1ca4018-30a6-4e85-92b5-e0dc500559b7"}
{"level":"info","ts":"2026-07-13T16:01:02.337Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c1ca4018-30a6-4e85-92b5-e0dc500559b7","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:01:02.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c1ca4018-30a6-4e85-92b5-e0dc500559b7","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:01:02.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c1ca4018-30a6-4e85-92b5-e0dc500559b7","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:02.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c1ca4018-30a6-4e85-92b5-e0dc500559b7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:01:02.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c1ca4018-30a6-4e85-92b5-e0dc500559b7","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:02.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c1ca4018-30a6-4e85-92b5-e0dc500559b7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:02.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c1ca4018-30a6-4e85-92b5-e0dc500559b7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:01:02.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c1ca4018-30a6-4e85-92b5-e0dc500559b7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:02.337Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c1ca4018-30a6-4e85-92b5-e0dc500559b7","podName":"fail-76mdf-head-rbw2b","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:02.337Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c1ca4018-30a6-4e85-92b5-e0dc500559b7","podName":"fail-76mdf-head-rbw2b","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:02.337Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c1ca4018-30a6-4e85-92b5-e0dc500559b7","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T16:01:02.337Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c1ca4018-30a6-4e85-92b5-e0dc500559b7","name":"fail-76mdf","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:01:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:01:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:01:02.337Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"0162c216-4b7e-43ba-b02c-8dbd1c5ed198","rayCluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:02.338Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"70ddb6d3-7abb-423f-871e-59d3c92d7d38","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:02.338Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"70ddb6d3-7abb-423f-871e-59d3c92d7d38","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:02.338Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"0162c216-4b7e-43ba-b02c-8dbd1c5ed198","rayCluster":"fail-76mdf","podIPs":[]}
{"level":"info","ts":"2026-07-13T16:01:02.338Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"70ddb6d3-7abb-423f-871e-59d3c92d7d38","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:02.338Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"70ddb6d3-7abb-423f-871e-59d3c92d7d38"}
{"level":"info","ts":"2026-07-13T16:01:02.338Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"70ddb6d3-7abb-423f-871e-59d3c92d7d38","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:02.338Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"1320040c-3795-4e0b-8684-871f6fb8def1","name":"fail-76mdf-workers"}
{"level":"info","ts":"2026-07-13T16:01:02.338Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"1320040c-3795-4e0b-8684-871f6fb8def1","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:02.338Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"e9ae9928-8f2a-44b9-8ec5-6543369cf280","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:02.346Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c1ca4018-30a6-4e85-92b5-e0dc500559b7","name":"fail-76mdf","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-76mdf\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-x5qvg","name":"fail-76mdf"}}
{"level":"info","ts":"2026-07-13T16:01:02.346Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c1ca4018-30a6-4e85-92b5-e0dc500559b7"}
{"level":"error","ts":"2026-07-13T16:01:02.346Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c1ca4018-30a6-4e85-92b5-e0dc500559b7","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-76mdf\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:01:02.350Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"e9ae9928-8f2a-44b9-8ec5-6543369cf280","name":"fail-76mdf-head"}
{"level":"info","ts":"2026-07-13T16:01:02.351Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"4f39f649-a030-4597-9135-f639f14c440f"}
{"level":"info","ts":"2026-07-13T16:01:02.351Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"4f39f649-a030-4597-9135-f639f14c440f"}
{"level":"info","ts":"2026-07-13T16:01:02.351Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"4f39f649-a030-4597-9135-f639f14c440f","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:01:02.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"4f39f649-a030-4597-9135-f639f14c440f","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:01:02.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"4f39f649-a030-4597-9135-f639f14c440f","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:02.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"4f39f649-a030-4597-9135-f639f14c440f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:01:02.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"4f39f649-a030-4597-9135-f639f14c440f","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:02.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"4f39f649-a030-4597-9135-f639f14c440f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:02.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"4f39f649-a030-4597-9135-f639f14c440f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:01:02.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"4f39f649-a030-4597-9135-f639f14c440f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:02.352Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"4f39f649-a030-4597-9135-f639f14c440f","podName":"fail-76mdf-small-group-worker-xzxgm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:02.352Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"4f39f649-a030-4597-9135-f639f14c440f","podName":"fail-76mdf-small-group-worker-xzxgm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:02.352Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"4f39f649-a030-4597-9135-f639f14c440f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:01:02.352Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"4f39f649-a030-4597-9135-f639f14c440f","seconds":300}
{"level":"info","ts":"2026-07-13T16:01:02.358Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"e9ae9928-8f2a-44b9-8ec5-6543369cf280","name":"fail-76mdf-workers"}
{"level":"info","ts":"2026-07-13T16:01:02.359Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"e9ae9928-8f2a-44b9-8ec5-6543369cf280","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:02.508Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-x5qvg"},"reconcileID":"88884c7c-4c5a-4e26-8201-93f3b368e9ea"}
{"level":"info","ts":"2026-07-13T16:01:02.567Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"1f4909fd-d768-4a19-b755-b8c85b03095d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:02.567Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"1f4909fd-d768-4a19-b755-b8c85b03095d","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:02.567Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"1f4909fd-d768-4a19-b755-b8c85b03095d","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:02.567Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"1f4909fd-d768-4a19-b755-b8c85b03095d"}
{"level":"info","ts":"2026-07-13T16:01:02.567Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"1f4909fd-d768-4a19-b755-b8c85b03095d","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:02.697Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"f3758ae8-0118-48be-a7ac-13bd93706a44"}
{"level":"info","ts":"2026-07-13T16:01:02.697Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"f3758ae8-0118-48be-a7ac-13bd93706a44"}
{"level":"info","ts":"2026-07-13T16:01:02.697Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"f3758ae8-0118-48be-a7ac-13bd93706a44","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:01:02.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"f3758ae8-0118-48be-a7ac-13bd93706a44","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:01:02.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"f3758ae8-0118-48be-a7ac-13bd93706a44","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:02.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"f3758ae8-0118-48be-a7ac-13bd93706a44","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:01:02.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"f3758ae8-0118-48be-a7ac-13bd93706a44","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:02.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"f3758ae8-0118-48be-a7ac-13bd93706a44","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:02.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"f3758ae8-0118-48be-a7ac-13bd93706a44","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:01:02.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"f3758ae8-0118-48be-a7ac-13bd93706a44","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:02.697Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"f3758ae8-0118-48be-a7ac-13bd93706a44","podName":"fail-76mdf-head-rbw2b","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:02.697Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"f3758ae8-0118-48be-a7ac-13bd93706a44","podName":"fail-76mdf-head-rbw2b","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:02.697Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"f3758ae8-0118-48be-a7ac-13bd93706a44","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:01:02.697Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"f3758ae8-0118-48be-a7ac-13bd93706a44","seconds":300}
{"level":"info","ts":"2026-07-13T16:01:02.727Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9b99bf3f-5c09-4a5d-a65d-3dd0b93eb36d"}
{"level":"info","ts":"2026-07-13T16:01:02.727Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9b99bf3f-5c09-4a5d-a65d-3dd0b93eb36d"}
{"level":"info","ts":"2026-07-13T16:01:02.727Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9b99bf3f-5c09-4a5d-a65d-3dd0b93eb36d","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:01:02.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9b99bf3f-5c09-4a5d-a65d-3dd0b93eb36d","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:01:02.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9b99bf3f-5c09-4a5d-a65d-3dd0b93eb36d","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:02.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9b99bf3f-5c09-4a5d-a65d-3dd0b93eb36d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:01:02.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9b99bf3f-5c09-4a5d-a65d-3dd0b93eb36d","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:02.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9b99bf3f-5c09-4a5d-a65d-3dd0b93eb36d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:02.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9b99bf3f-5c09-4a5d-a65d-3dd0b93eb36d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:01:02.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9b99bf3f-5c09-4a5d-a65d-3dd0b93eb36d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:02.727Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9b99bf3f-5c09-4a5d-a65d-3dd0b93eb36d","podName":"fail-76mdf-head-rbw2b","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:02.727Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9b99bf3f-5c09-4a5d-a65d-3dd0b93eb36d","podName":"fail-76mdf-head-rbw2b","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:02.727Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9b99bf3f-5c09-4a5d-a65d-3dd0b93eb36d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:01:02.727Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9b99bf3f-5c09-4a5d-a65d-3dd0b93eb36d","seconds":300}
{"level":"info","ts":"2026-07-13T16:01:03.444Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"d1de99be-c06d-4e17-bd68-0af44e45b8f8"}
{"level":"info","ts":"2026-07-13T16:01:03.444Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"d1de99be-c06d-4e17-bd68-0af44e45b8f8"}
{"level":"info","ts":"2026-07-13T16:01:03.444Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"d1de99be-c06d-4e17-bd68-0af44e45b8f8","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:01:03.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"d1de99be-c06d-4e17-bd68-0af44e45b8f8","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:01:03.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"d1de99be-c06d-4e17-bd68-0af44e45b8f8","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:03.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"d1de99be-c06d-4e17-bd68-0af44e45b8f8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:01:03.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"d1de99be-c06d-4e17-bd68-0af44e45b8f8","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:03.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"d1de99be-c06d-4e17-bd68-0af44e45b8f8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:03.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"d1de99be-c06d-4e17-bd68-0af44e45b8f8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:01:03.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"d1de99be-c06d-4e17-bd68-0af44e45b8f8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:03.444Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"d1de99be-c06d-4e17-bd68-0af44e45b8f8","podName":"fail-76mdf-head-rbw2b","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:03.444Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"d1de99be-c06d-4e17-bd68-0af44e45b8f8","podName":"fail-76mdf-head-rbw2b","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:03.445Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"d1de99be-c06d-4e17-bd68-0af44e45b8f8","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"newHead":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"}}
{"level":"info","ts":"2026-07-13T16:01:03.445Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"d1de99be-c06d-4e17-bd68-0af44e45b8f8","name":"fail-76mdf","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:01:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:01:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:01:03.459Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"77f24c3e-a8b9-4846-93d5-c60aab751616","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:03.460Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"2788bec6-26dd-4b49-b364-9a5424fe8779","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:03.460Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"2788bec6-26dd-4b49-b364-9a5424fe8779","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:03.460Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"2788bec6-26dd-4b49-b364-9a5424fe8779","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:03.460Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"2788bec6-26dd-4b49-b364-9a5424fe8779"}
{"level":"info","ts":"2026-07-13T16:01:03.460Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"2788bec6-26dd-4b49-b364-9a5424fe8779","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:03.521Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"91b4ff15-26c1-4d6f-b4e9-c15ee908bce1","rayCluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:03.522Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"91b4ff15-26c1-4d6f-b4e9-c15ee908bce1","pod":"fail-76mdf-head-rbw2b","ip":"10.133.0.30"}
{"level":"info","ts":"2026-07-13T16:01:03.526Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"77f24c3e-a8b9-4846-93d5-c60aab751616","name":"fail-76mdf-head"}
{"level":"info","ts":"2026-07-13T16:01:03.537Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"91b4ff15-26c1-4d6f-b4e9-c15ee908bce1","certificate":"ray-head-cert-fail-76mdf","dnsNames":["fail-76mdf-head-svc","localhost","fail-76mdf-head-svc.test-ns-x5qvg.svc","fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local"],"ipAddresses":["10.133.0.30","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:01:03.621Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"77f24c3e-a8b9-4846-93d5-c60aab751616","name":"fail-76mdf-workers"}
{"level":"info","ts":"2026-07-13T16:01:03.622Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"77f24c3e-a8b9-4846-93d5-c60aab751616","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:03.622Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"91b4ff15-26c1-4d6f-b4e9-c15ee908bce1","certificate":"ray-worker-cert-fail-76mdf","dnsNames":["fail-76mdf-worker-svc","localhost","fail-76mdf-small-group","fail-76mdf-worker-svc.test-ns-x5qvg.svc","fail-76mdf-worker-svc.test-ns-x5qvg.svc.cluster.local","*.fail-76mdf-worker-svc.test-ns-x5qvg.svc","*.fail-76mdf-worker-svc.test-ns-x5qvg.svc.cluster.local","*-worker-*.test-ns-x5qvg.svc","*-worker-*.test-ns-x5qvg.svc.cluster.local"],"ipAddresses":["10.133.0.30","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:01:03.622Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"91b4ff15-26c1-4d6f-b4e9-c15ee908bce1","rayCluster":"fail-76mdf","podIPs":["10.133.0.30"]}
{"level":"info","ts":"2026-07-13T16:01:03.724Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"1aadb102-7046-4c6d-a12a-ecd52b98a049"}
{"level":"info","ts":"2026-07-13T16:01:03.724Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"1aadb102-7046-4c6d-a12a-ecd52b98a049"}
{"level":"info","ts":"2026-07-13T16:01:03.724Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"1aadb102-7046-4c6d-a12a-ecd52b98a049","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:01:03.724Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"1aadb102-7046-4c6d-a12a-ecd52b98a049","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:01:03.724Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"1aadb102-7046-4c6d-a12a-ecd52b98a049","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:03.724Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"1aadb102-7046-4c6d-a12a-ecd52b98a049","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:01:03.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"1aadb102-7046-4c6d-a12a-ecd52b98a049","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:03.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"1aadb102-7046-4c6d-a12a-ecd52b98a049","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:03.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"1aadb102-7046-4c6d-a12a-ecd52b98a049","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:01:03.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"1aadb102-7046-4c6d-a12a-ecd52b98a049","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:03.725Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"1aadb102-7046-4c6d-a12a-ecd52b98a049","podName":"fail-76mdf-head-rbw2b","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:03.725Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"1aadb102-7046-4c6d-a12a-ecd52b98a049","podName":"fail-76mdf-head-rbw2b","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:03.725Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"1aadb102-7046-4c6d-a12a-ecd52b98a049","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:01:03.725Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"1aadb102-7046-4c6d-a12a-ecd52b98a049","seconds":300}
{"level":"info","ts":"2026-07-13T16:01:05.522Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"38994a25-6742-4ab1-844d-5b2c52d95f45"}
{"level":"info","ts":"2026-07-13T16:01:05.522Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"38994a25-6742-4ab1-844d-5b2c52d95f45"}
{"level":"info","ts":"2026-07-13T16:01:05.522Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"38994a25-6742-4ab1-844d-5b2c52d95f45","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:01:05.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"38994a25-6742-4ab1-844d-5b2c52d95f45","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:01:05.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"38994a25-6742-4ab1-844d-5b2c52d95f45","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:05.523Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"38994a25-6742-4ab1-844d-5b2c52d95f45","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:01:05.523Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"38994a25-6742-4ab1-844d-5b2c52d95f45","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:05.523Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"38994a25-6742-4ab1-844d-5b2c52d95f45","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:05.523Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"38994a25-6742-4ab1-844d-5b2c52d95f45","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:01:05.523Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"38994a25-6742-4ab1-844d-5b2c52d95f45","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:05.523Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"38994a25-6742-4ab1-844d-5b2c52d95f45","podName":"fail-76mdf-head-rbw2b","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:05.523Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"38994a25-6742-4ab1-844d-5b2c52d95f45","podName":"fail-76mdf-head-rbw2b","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:05.523Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"38994a25-6742-4ab1-844d-5b2c52d95f45","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:01:05.523Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"38994a25-6742-4ab1-844d-5b2c52d95f45","seconds":300}
{"level":"info","ts":"2026-07-13T16:01:05.569Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"0bf8ac50-979d-4701-b976-0a88361a52e9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:05.569Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"0bf8ac50-979d-4701-b976-0a88361a52e9","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:05.569Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"0bf8ac50-979d-4701-b976-0a88361a52e9","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:05.569Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"0bf8ac50-979d-4701-b976-0a88361a52e9"}
{"level":"info","ts":"2026-07-13T16:01:05.569Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"0bf8ac50-979d-4701-b976-0a88361a52e9","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:08.464Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9feaa3a3-c555-40d6-87a1-80fa8e1315e8"}
{"level":"info","ts":"2026-07-13T16:01:08.464Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9feaa3a3-c555-40d6-87a1-80fa8e1315e8"}
{"level":"info","ts":"2026-07-13T16:01:08.464Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9feaa3a3-c555-40d6-87a1-80fa8e1315e8","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:01:08.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9feaa3a3-c555-40d6-87a1-80fa8e1315e8","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:01:08.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9feaa3a3-c555-40d6-87a1-80fa8e1315e8","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:08.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9feaa3a3-c555-40d6-87a1-80fa8e1315e8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:01:08.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9feaa3a3-c555-40d6-87a1-80fa8e1315e8","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:08.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9feaa3a3-c555-40d6-87a1-80fa8e1315e8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:08.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9feaa3a3-c555-40d6-87a1-80fa8e1315e8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:01:08.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9feaa3a3-c555-40d6-87a1-80fa8e1315e8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:08.464Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9feaa3a3-c555-40d6-87a1-80fa8e1315e8","podName":"fail-76mdf-head-rbw2b","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:08.464Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9feaa3a3-c555-40d6-87a1-80fa8e1315e8","podName":"fail-76mdf-head-rbw2b","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:08.464Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9feaa3a3-c555-40d6-87a1-80fa8e1315e8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:01:08.464Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"9feaa3a3-c555-40d6-87a1-80fa8e1315e8","seconds":300}
{"level":"info","ts":"2026-07-13T16:01:08.570Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"0d867f31-b992-41b1-a5b1-bc9cb6ad4e35","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:08.570Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"0d867f31-b992-41b1-a5b1-bc9cb6ad4e35","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:08.570Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"0d867f31-b992-41b1-a5b1-bc9cb6ad4e35","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:08.570Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"0d867f31-b992-41b1-a5b1-bc9cb6ad4e35"}
{"level":"info","ts":"2026-07-13T16:01:08.570Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"0d867f31-b992-41b1-a5b1-bc9cb6ad4e35","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:09.473Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"8f3f106c-ad91-4e18-b64b-754abd2db3e0"}
{"level":"info","ts":"2026-07-13T16:01:09.473Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"8f3f106c-ad91-4e18-b64b-754abd2db3e0"}
{"level":"info","ts":"2026-07-13T16:01:09.473Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"8f3f106c-ad91-4e18-b64b-754abd2db3e0","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:01:09.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"8f3f106c-ad91-4e18-b64b-754abd2db3e0","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:01:09.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"8f3f106c-ad91-4e18-b64b-754abd2db3e0","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:09.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"8f3f106c-ad91-4e18-b64b-754abd2db3e0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:01:09.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"8f3f106c-ad91-4e18-b64b-754abd2db3e0","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:09.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"8f3f106c-ad91-4e18-b64b-754abd2db3e0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:09.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"8f3f106c-ad91-4e18-b64b-754abd2db3e0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:01:09.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"8f3f106c-ad91-4e18-b64b-754abd2db3e0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:09.473Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"8f3f106c-ad91-4e18-b64b-754abd2db3e0","podName":"fail-76mdf-small-group-worker-xzxgm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:09.473Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"8f3f106c-ad91-4e18-b64b-754abd2db3e0","podName":"fail-76mdf-small-group-worker-xzxgm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:09.473Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"8f3f106c-ad91-4e18-b64b-754abd2db3e0","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T16:01:09.473Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"8f3f106c-ad91-4e18-b64b-754abd2db3e0","name":"fail-76mdf","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:01:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:01:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:01:09.486Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"24d2022b-bca8-4254-9327-ff6914725790","rayCluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:09.486Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"3b3c2555-d96a-4794-aaf5-054d85eeac62","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:09.486Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"24d2022b-bca8-4254-9327-ff6914725790","pod":"fail-76mdf-small-group-worker-xzxgm","ip":"10.134.0.32"}
{"level":"info","ts":"2026-07-13T16:01:09.486Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"24d2022b-bca8-4254-9327-ff6914725790","pod":"fail-76mdf-head-rbw2b","ip":"10.133.0.30"}
{"level":"info","ts":"2026-07-13T16:01:09.486Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"61506419-2f09-45e7-ab95-a8d21248e4d3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:09.486Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"61506419-2f09-45e7-ab95-a8d21248e4d3","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:09.486Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"61506419-2f09-45e7-ab95-a8d21248e4d3","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:09.486Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"61506419-2f09-45e7-ab95-a8d21248e4d3"}
{"level":"info","ts":"2026-07-13T16:01:09.486Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"61506419-2f09-45e7-ab95-a8d21248e4d3","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:09.498Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"24d2022b-bca8-4254-9327-ff6914725790","certificate":"ray-head-cert-fail-76mdf","dnsNames":["fail-76mdf-head-svc","localhost","fail-76mdf-head-svc.test-ns-x5qvg.svc","fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local"],"ipAddresses":["10.134.0.32","10.133.0.30","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:01:09.504Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"3b3c2555-d96a-4794-aaf5-054d85eeac62","name":"fail-76mdf-head"}
{"level":"info","ts":"2026-07-13T16:01:09.510Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"24d2022b-bca8-4254-9327-ff6914725790","certificate":"ray-worker-cert-fail-76mdf","dnsNames":["fail-76mdf-worker-svc","localhost","fail-76mdf-small-group","fail-76mdf-worker-svc.test-ns-x5qvg.svc","fail-76mdf-worker-svc.test-ns-x5qvg.svc.cluster.local","*.fail-76mdf-worker-svc.test-ns-x5qvg.svc","*.fail-76mdf-worker-svc.test-ns-x5qvg.svc.cluster.local","*-worker-*.test-ns-x5qvg.svc","*-worker-*.test-ns-x5qvg.svc.cluster.local"],"ipAddresses":["10.134.0.32","10.133.0.30","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:01:09.510Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"24d2022b-bca8-4254-9327-ff6914725790","rayCluster":"fail-76mdf","podIPs":["10.134.0.32","10.133.0.30"]}
{"level":"info","ts":"2026-07-13T16:01:09.516Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"3b3c2555-d96a-4794-aaf5-054d85eeac62","name":"fail-76mdf-workers"}
{"level":"info","ts":"2026-07-13T16:01:09.516Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"3b3c2555-d96a-4794-aaf5-054d85eeac62","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:11.486Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"979861b7-e1e2-44c8-98be-80bd79829d1f"}
{"level":"info","ts":"2026-07-13T16:01:11.486Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"979861b7-e1e2-44c8-98be-80bd79829d1f"}
{"level":"info","ts":"2026-07-13T16:01:11.486Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"979861b7-e1e2-44c8-98be-80bd79829d1f","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:01:11.486Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"979861b7-e1e2-44c8-98be-80bd79829d1f","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:01:11.486Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"979861b7-e1e2-44c8-98be-80bd79829d1f","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:11.486Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"979861b7-e1e2-44c8-98be-80bd79829d1f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:01:11.486Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"979861b7-e1e2-44c8-98be-80bd79829d1f","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:11.486Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"979861b7-e1e2-44c8-98be-80bd79829d1f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:11.486Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"979861b7-e1e2-44c8-98be-80bd79829d1f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:01:11.486Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"979861b7-e1e2-44c8-98be-80bd79829d1f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:11.486Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"979861b7-e1e2-44c8-98be-80bd79829d1f","podName":"fail-76mdf-head-rbw2b","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:01:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:01:11.486Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"979861b7-e1e2-44c8-98be-80bd79829d1f","podName":"fail-76mdf-head-rbw2b","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:01:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:01:11.487Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"979861b7-e1e2-44c8-98be-80bd79829d1f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:01:11.487Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"979861b7-e1e2-44c8-98be-80bd79829d1f","seconds":300}
{"level":"info","ts":"2026-07-13T16:01:11.571Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"bc4eb7b0-e458-4b49-ad82-f77ab82ceeee","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:11.571Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"bc4eb7b0-e458-4b49-ad82-f77ab82ceeee","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:11.571Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"bc4eb7b0-e458-4b49-ad82-f77ab82ceeee","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:11.571Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"bc4eb7b0-e458-4b49-ad82-f77ab82ceeee"}
{"level":"info","ts":"2026-07-13T16:01:11.571Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"bc4eb7b0-e458-4b49-ad82-f77ab82ceeee","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:14.572Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"bcd4d0ff-aa58-468a-9c2b-bf38e021ca87","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:14.572Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"bcd4d0ff-aa58-468a-9c2b-bf38e021ca87","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:14.572Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"bcd4d0ff-aa58-468a-9c2b-bf38e021ca87","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:14.572Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"bcd4d0ff-aa58-468a-9c2b-bf38e021ca87"}
{"level":"info","ts":"2026-07-13T16:01:14.572Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"bcd4d0ff-aa58-468a-9c2b-bf38e021ca87","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:17.574Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"427493b0-3519-4ccc-8c02-2c4f0920fb7a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:17.574Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"427493b0-3519-4ccc-8c02-2c4f0920fb7a","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:17.574Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"427493b0-3519-4ccc-8c02-2c4f0920fb7a","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:17.574Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"427493b0-3519-4ccc-8c02-2c4f0920fb7a"}
{"level":"info","ts":"2026-07-13T16:01:17.574Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"427493b0-3519-4ccc-8c02-2c4f0920fb7a","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:20.503Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"48efecfb-0fff-4cf7-8a4b-5fa19abe84ee"}
{"level":"info","ts":"2026-07-13T16:01:20.503Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"48efecfb-0fff-4cf7-8a4b-5fa19abe84ee"}
{"level":"info","ts":"2026-07-13T16:01:20.503Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"48efecfb-0fff-4cf7-8a4b-5fa19abe84ee","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:01:20.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"48efecfb-0fff-4cf7-8a4b-5fa19abe84ee","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:01:20.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"48efecfb-0fff-4cf7-8a4b-5fa19abe84ee","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:20.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"48efecfb-0fff-4cf7-8a4b-5fa19abe84ee","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:01:20.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"48efecfb-0fff-4cf7-8a4b-5fa19abe84ee","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:20.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"48efecfb-0fff-4cf7-8a4b-5fa19abe84ee","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:20.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"48efecfb-0fff-4cf7-8a4b-5fa19abe84ee","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:01:20.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"48efecfb-0fff-4cf7-8a4b-5fa19abe84ee","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:20.503Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"48efecfb-0fff-4cf7-8a4b-5fa19abe84ee","podName":"fail-76mdf-small-group-worker-xzxgm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:20.503Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"48efecfb-0fff-4cf7-8a4b-5fa19abe84ee","podName":"fail-76mdf-small-group-worker-xzxgm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:20.503Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"48efecfb-0fff-4cf7-8a4b-5fa19abe84ee","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T16:01:20.504Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"48efecfb-0fff-4cf7-8a4b-5fa19abe84ee","name":"fail-76mdf","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:01:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:01:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:01:20.520Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"99213c93-c856-4758-97e9-ff6b001adfe0","rayCluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:20.520Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"7573c215-c8f1-4d4b-aadf-d889c9e23e52","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:20.520Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"99213c93-c856-4758-97e9-ff6b001adfe0","pod":"fail-76mdf-small-group-worker-xzxgm","ip":"10.134.0.32"}
{"level":"info","ts":"2026-07-13T16:01:20.520Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"99213c93-c856-4758-97e9-ff6b001adfe0","pod":"fail-76mdf-head-rbw2b","ip":"10.133.0.30"}
{"level":"info","ts":"2026-07-13T16:01:20.520Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"99213c93-c856-4758-97e9-ff6b001adfe0","rayCluster":"fail-76mdf","podIPs":["10.134.0.32","10.133.0.30"]}
{"level":"info","ts":"2026-07-13T16:01:20.520Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"9f358ab2-8077-451f-8a34-95f143fdfcf8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:20.520Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"9f358ab2-8077-451f-8a34-95f143fdfcf8","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:20.520Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"9f358ab2-8077-451f-8a34-95f143fdfcf8","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:20.520Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"9f358ab2-8077-451f-8a34-95f143fdfcf8"}
{"level":"info","ts":"2026-07-13T16:01:20.520Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"9f358ab2-8077-451f-8a34-95f143fdfcf8","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:20.536Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"7573c215-c8f1-4d4b-aadf-d889c9e23e52","name":"fail-76mdf-head"}
{"level":"info","ts":"2026-07-13T16:01:20.549Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"7573c215-c8f1-4d4b-aadf-d889c9e23e52","name":"fail-76mdf-workers"}
{"level":"info","ts":"2026-07-13T16:01:20.549Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"7573c215-c8f1-4d4b-aadf-d889c9e23e52","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:20.575Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3bd98a24-f8cc-448b-88ef-21d053a54b21","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:20.575Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3bd98a24-f8cc-448b-88ef-21d053a54b21","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:20.575Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3bd98a24-f8cc-448b-88ef-21d053a54b21","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:20.575Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3bd98a24-f8cc-448b-88ef-21d053a54b21"}
{"level":"info","ts":"2026-07-13T16:01:20.575Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3bd98a24-f8cc-448b-88ef-21d053a54b21","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:22.520Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"262473b7-63c4-4750-a65c-0178c9f224c1"}
{"level":"info","ts":"2026-07-13T16:01:22.520Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"262473b7-63c4-4750-a65c-0178c9f224c1"}
{"level":"info","ts":"2026-07-13T16:01:22.520Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"262473b7-63c4-4750-a65c-0178c9f224c1","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:01:22.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"262473b7-63c4-4750-a65c-0178c9f224c1","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:01:22.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"262473b7-63c4-4750-a65c-0178c9f224c1","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:22.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"262473b7-63c4-4750-a65c-0178c9f224c1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:01:22.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"262473b7-63c4-4750-a65c-0178c9f224c1","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:01:22.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"262473b7-63c4-4750-a65c-0178c9f224c1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:22.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"262473b7-63c4-4750-a65c-0178c9f224c1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:01:22.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"262473b7-63c4-4750-a65c-0178c9f224c1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:01:22.520Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"262473b7-63c4-4750-a65c-0178c9f224c1","podName":"fail-76mdf-small-group-worker-xzxgm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:22.520Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"262473b7-63c4-4750-a65c-0178c9f224c1","podName":"fail-76mdf-small-group-worker-xzxgm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:01:22.520Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"262473b7-63c4-4750-a65c-0178c9f224c1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:01:22.520Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"262473b7-63c4-4750-a65c-0178c9f224c1","seconds":300}
{"level":"info","ts":"2026-07-13T16:01:23.576Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cbbcccb4-297a-42e3-8ece-938663de5cb0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:23.576Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cbbcccb4-297a-42e3-8ece-938663de5cb0","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:23.576Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cbbcccb4-297a-42e3-8ece-938663de5cb0","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:23.576Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cbbcccb4-297a-42e3-8ece-938663de5cb0"}
{"level":"info","ts":"2026-07-13T16:01:23.576Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cbbcccb4-297a-42e3-8ece-938663de5cb0","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:26.577Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"4f7c6d8f-3b86-451e-8db8-61932fa996e6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:26.577Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"4f7c6d8f-3b86-451e-8db8-61932fa996e6","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:26.577Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"4f7c6d8f-3b86-451e-8db8-61932fa996e6","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:26.577Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"4f7c6d8f-3b86-451e-8db8-61932fa996e6"}
{"level":"info","ts":"2026-07-13T16:01:26.577Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"4f7c6d8f-3b86-451e-8db8-61932fa996e6","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:29.577Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"be45f601-4518-4075-8696-41610dd77592","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:29.577Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"be45f601-4518-4075-8696-41610dd77592","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:29.577Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"be45f601-4518-4075-8696-41610dd77592","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:29.578Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"be45f601-4518-4075-8696-41610dd77592"}
{"level":"info","ts":"2026-07-13T16:01:29.578Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"be45f601-4518-4075-8696-41610dd77592","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:29.822Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"a1e2cf3f-ee4d-4726-944c-8eb4830730c0","rayCluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:29.822Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"a1e2cf3f-ee4d-4726-944c-8eb4830730c0","pod":"fail-76mdf-head-rbw2b","ip":"10.133.0.30"}
{"level":"info","ts":"2026-07-13T16:01:29.822Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"a1e2cf3f-ee4d-4726-944c-8eb4830730c0","pod":"fail-76mdf-small-group-worker-xzxgm","ip":"10.134.0.32"}
{"level":"info","ts":"2026-07-13T16:01:29.822Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"a1e2cf3f-ee4d-4726-944c-8eb4830730c0","rayCluster":"fail-76mdf","podIPs":["10.133.0.30","10.134.0.32"]}
{"level":"info","ts":"2026-07-13T16:01:32.578Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3b8e99cd-17ee-459e-a657-84a9c6acbeb4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:32.578Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3b8e99cd-17ee-459e-a657-84a9c6acbeb4","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:32.578Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3b8e99cd-17ee-459e-a657-84a9c6acbeb4","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:32.578Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3b8e99cd-17ee-459e-a657-84a9c6acbeb4"}
{"level":"info","ts":"2026-07-13T16:01:32.578Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3b8e99cd-17ee-459e-a657-84a9c6acbeb4","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:34.943Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"894615df-f7e0-4977-949c-8b3b845264bf","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:01:34.943Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"894615df-f7e0-4977-949c-8b3b845264bf","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T16:01:34.943Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"894615df-f7e0-4977-949c-8b3b845264bf","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T16:01:34.943Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"894615df-f7e0-4977-949c-8b3b845264bf","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T16:01:35.579Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"effc24dd-0b92-4541-9ca5-93890e7431c8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:35.579Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"effc24dd-0b92-4541-9ca5-93890e7431c8","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:35.579Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"effc24dd-0b92-4541-9ca5-93890e7431c8","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:35.579Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"effc24dd-0b92-4541-9ca5-93890e7431c8"}
{"level":"info","ts":"2026-07-13T16:01:35.579Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"effc24dd-0b92-4541-9ca5-93890e7431c8","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:38.580Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"044afa46-de0c-41ae-b20d-0a341fa45d9d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:38.580Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"044afa46-de0c-41ae-b20d-0a341fa45d9d","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:38.580Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"044afa46-de0c-41ae-b20d-0a341fa45d9d","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:38.580Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"044afa46-de0c-41ae-b20d-0a341fa45d9d"}
{"level":"info","ts":"2026-07-13T16:01:38.580Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"044afa46-de0c-41ae-b20d-0a341fa45d9d","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:41.580Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"40c7653c-984e-4db9-a198-b1cb944f2e13","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:41.580Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"40c7653c-984e-4db9-a198-b1cb944f2e13","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:41.580Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"40c7653c-984e-4db9-a198-b1cb944f2e13","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:41.581Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"40c7653c-984e-4db9-a198-b1cb944f2e13"}
{"level":"info","ts":"2026-07-13T16:01:41.581Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"40c7653c-984e-4db9-a198-b1cb944f2e13","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:44.581Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cc234053-8a72-4643-9366-2f59f5d712db","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:44.581Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cc234053-8a72-4643-9366-2f59f5d712db","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:44.581Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cc234053-8a72-4643-9366-2f59f5d712db","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:44.581Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cc234053-8a72-4643-9366-2f59f5d712db"}
{"level":"info","ts":"2026-07-13T16:01:44.581Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cc234053-8a72-4643-9366-2f59f5d712db","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:47.582Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"14b3772b-bdc4-4059-a683-b3c886322cc9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:47.582Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"14b3772b-bdc4-4059-a683-b3c886322cc9","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:47.582Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"14b3772b-bdc4-4059-a683-b3c886322cc9","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:47.582Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"14b3772b-bdc4-4059-a683-b3c886322cc9"}
{"level":"info","ts":"2026-07-13T16:01:47.582Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"14b3772b-bdc4-4059-a683-b3c886322cc9","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:50.583Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"5bc48c76-da90-40b8-9d9b-53d905905ffc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:50.583Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"5bc48c76-da90-40b8-9d9b-53d905905ffc","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:50.583Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"5bc48c76-da90-40b8-9d9b-53d905905ffc","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:50.584Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"5bc48c76-da90-40b8-9d9b-53d905905ffc"}
{"level":"info","ts":"2026-07-13T16:01:50.584Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"5bc48c76-da90-40b8-9d9b-53d905905ffc","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:53.584Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"635a2d08-dee8-4961-b289-6a00795ef8d2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:53.584Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"635a2d08-dee8-4961-b289-6a00795ef8d2","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:53.584Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"635a2d08-dee8-4961-b289-6a00795ef8d2","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:53.584Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"635a2d08-dee8-4961-b289-6a00795ef8d2"}
{"level":"info","ts":"2026-07-13T16:01:53.584Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"635a2d08-dee8-4961-b289-6a00795ef8d2","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:56.585Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7986140d-cad6-4b92-9f95-270de5563597","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:56.585Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7986140d-cad6-4b92-9f95-270de5563597","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:56.585Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7986140d-cad6-4b92-9f95-270de5563597","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:56.585Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7986140d-cad6-4b92-9f95-270de5563597"}
{"level":"info","ts":"2026-07-13T16:01:56.585Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7986140d-cad6-4b92-9f95-270de5563597","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:01:59.586Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cc327fc9-69d3-4dc8-b214-498c04c5679a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:01:59.586Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cc327fc9-69d3-4dc8-b214-498c04c5679a","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:01:59.586Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cc327fc9-69d3-4dc8-b214-498c04c5679a","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:01:59.586Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cc327fc9-69d3-4dc8-b214-498c04c5679a"}
{"level":"info","ts":"2026-07-13T16:01:59.586Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cc327fc9-69d3-4dc8-b214-498c04c5679a","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:02:02.587Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"4a976415-0727-4215-bbda-16cba213a234","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:02.587Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"4a976415-0727-4215-bbda-16cba213a234","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:02.587Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"4a976415-0727-4215-bbda-16cba213a234","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:02.587Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"4a976415-0727-4215-bbda-16cba213a234"}
{"level":"info","ts":"2026-07-13T16:02:02.587Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"4a976415-0727-4215-bbda-16cba213a234","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:02:05.588Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3edaef97-a90a-4f40-8fed-e60633f31ad8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:05.588Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3edaef97-a90a-4f40-8fed-e60633f31ad8","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:05.588Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3edaef97-a90a-4f40-8fed-e60633f31ad8","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:05.588Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3edaef97-a90a-4f40-8fed-e60633f31ad8"}
{"level":"info","ts":"2026-07-13T16:02:05.588Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3edaef97-a90a-4f40-8fed-e60633f31ad8","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:02:08.589Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"84bc313c-bc37-4aef-8be5-0692e1b3697f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:08.589Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"84bc313c-bc37-4aef-8be5-0692e1b3697f","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:08.589Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"84bc313c-bc37-4aef-8be5-0692e1b3697f","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:08.589Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"84bc313c-bc37-4aef-8be5-0692e1b3697f"}
{"level":"info","ts":"2026-07-13T16:02:08.589Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"84bc313c-bc37-4aef-8be5-0692e1b3697f","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:02:11.590Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"2627f086-4c16-413d-85de-217614381d59","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:11.590Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"2627f086-4c16-413d-85de-217614381d59","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:11.590Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"2627f086-4c16-413d-85de-217614381d59","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:11.590Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"2627f086-4c16-413d-85de-217614381d59"}
{"level":"info","ts":"2026-07-13T16:02:11.590Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"2627f086-4c16-413d-85de-217614381d59","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:02:14.591Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"1c905bf1-7d32-4e34-a4b4-8cc37b003e2a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:14.591Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"1c905bf1-7d32-4e34-a4b4-8cc37b003e2a","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:14.591Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"1c905bf1-7d32-4e34-a4b4-8cc37b003e2a","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:14.591Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"1c905bf1-7d32-4e34-a4b4-8cc37b003e2a"}
{"level":"info","ts":"2026-07-13T16:02:14.591Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"1c905bf1-7d32-4e34-a4b4-8cc37b003e2a","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:02:17.592Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"ae6a8c31-76fd-49ae-9b71-6fb6ac74d8f1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:17.592Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"ae6a8c31-76fd-49ae-9b71-6fb6ac74d8f1","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:17.592Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"ae6a8c31-76fd-49ae-9b71-6fb6ac74d8f1","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:17.592Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"ae6a8c31-76fd-49ae-9b71-6fb6ac74d8f1"}
{"level":"info","ts":"2026-07-13T16:02:17.592Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"ae6a8c31-76fd-49ae-9b71-6fb6ac74d8f1","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:02:20.593Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3ceb676d-3f7f-427d-8526-8f19dc24a455","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:20.593Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3ceb676d-3f7f-427d-8526-8f19dc24a455","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:20.593Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3ceb676d-3f7f-427d-8526-8f19dc24a455","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:20.593Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3ceb676d-3f7f-427d-8526-8f19dc24a455"}
{"level":"info","ts":"2026-07-13T16:02:20.593Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3ceb676d-3f7f-427d-8526-8f19dc24a455","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:02:23.594Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"fc7bfd41-27f2-454a-aa67-d26838abe6c3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:23.594Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"fc7bfd41-27f2-454a-aa67-d26838abe6c3","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:23.594Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"fc7bfd41-27f2-454a-aa67-d26838abe6c3","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:23.594Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"fc7bfd41-27f2-454a-aa67-d26838abe6c3"}
{"level":"info","ts":"2026-07-13T16:02:23.594Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"fc7bfd41-27f2-454a-aa67-d26838abe6c3","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:02:24.045Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"bcc0d10c-93e8-4853-ad13-a3db4e4f906b"}
{"level":"info","ts":"2026-07-13T16:02:24.045Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"bcc0d10c-93e8-4853-ad13-a3db4e4f906b"}
{"level":"info","ts":"2026-07-13T16:02:24.045Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"bcc0d10c-93e8-4853-ad13-a3db4e4f906b","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:02:24.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"bcc0d10c-93e8-4853-ad13-a3db4e4f906b","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:02:24.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"bcc0d10c-93e8-4853-ad13-a3db4e4f906b","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:02:24.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"bcc0d10c-93e8-4853-ad13-a3db4e4f906b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:02:24.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"bcc0d10c-93e8-4853-ad13-a3db4e4f906b","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:02:24.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"bcc0d10c-93e8-4853-ad13-a3db4e4f906b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:02:24.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"bcc0d10c-93e8-4853-ad13-a3db4e4f906b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:02:24.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"bcc0d10c-93e8-4853-ad13-a3db4e4f906b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:02:24.046Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"bcc0d10c-93e8-4853-ad13-a3db4e4f906b","podName":"fail-76mdf-small-group-worker-xzxgm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:02:24.046Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"bcc0d10c-93e8-4853-ad13-a3db4e4f906b","podName":"fail-76mdf-small-group-worker-xzxgm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:02:24.046Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"bcc0d10c-93e8-4853-ad13-a3db4e4f906b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:02:24.046Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"bcc0d10c-93e8-4853-ad13-a3db4e4f906b","seconds":300}
{"level":"info","ts":"2026-07-13T16:02:25.051Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"ed225fa8-ec45-429f-88f0-4824b9386b6b"}
{"level":"info","ts":"2026-07-13T16:02:25.051Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"ed225fa8-ec45-429f-88f0-4824b9386b6b"}
{"level":"info","ts":"2026-07-13T16:02:25.051Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"ed225fa8-ec45-429f-88f0-4824b9386b6b","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:02:25.051Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"ed225fa8-ec45-429f-88f0-4824b9386b6b","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:02:25.051Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"ed225fa8-ec45-429f-88f0-4824b9386b6b","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:02:25.051Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"ed225fa8-ec45-429f-88f0-4824b9386b6b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:02:25.051Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"ed225fa8-ec45-429f-88f0-4824b9386b6b","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:02:25.051Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"ed225fa8-ec45-429f-88f0-4824b9386b6b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:02:25.051Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"ed225fa8-ec45-429f-88f0-4824b9386b6b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:02:25.051Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"ed225fa8-ec45-429f-88f0-4824b9386b6b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:02:25.051Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"ed225fa8-ec45-429f-88f0-4824b9386b6b","podName":"fail-76mdf-small-group-worker-xzxgm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:02:25.051Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"ed225fa8-ec45-429f-88f0-4824b9386b6b","podName":"fail-76mdf-small-group-worker-xzxgm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:02:25.051Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"ed225fa8-ec45-429f-88f0-4824b9386b6b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:02:25.051Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"ed225fa8-ec45-429f-88f0-4824b9386b6b","seconds":300}
{"level":"info","ts":"2026-07-13T16:02:26.594Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d44a1644-4795-44ed-a096-825c3ea06445","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:26.594Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d44a1644-4795-44ed-a096-825c3ea06445","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:26.594Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d44a1644-4795-44ed-a096-825c3ea06445","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:26.595Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d44a1644-4795-44ed-a096-825c3ea06445"}
{"level":"info","ts":"2026-07-13T16:02:26.595Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d44a1644-4795-44ed-a096-825c3ea06445","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:02:29.595Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"ab0afafd-1946-470c-b464-49a41cf5a047","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:29.595Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"ab0afafd-1946-470c-b464-49a41cf5a047","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:29.595Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"ab0afafd-1946-470c-b464-49a41cf5a047","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:29.595Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"ab0afafd-1946-470c-b464-49a41cf5a047"}
{"level":"info","ts":"2026-07-13T16:02:29.595Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"ab0afafd-1946-470c-b464-49a41cf5a047","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:02:29.823Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"b1411084-ce07-415f-a6d6-bfa697b781a6","rayCluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:29.823Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"b1411084-ce07-415f-a6d6-bfa697b781a6","pod":"fail-76mdf-head-rbw2b","ip":"10.133.0.30"}
{"level":"info","ts":"2026-07-13T16:02:29.823Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"b1411084-ce07-415f-a6d6-bfa697b781a6","pod":"fail-76mdf-small-group-worker-xzxgm","ip":"10.134.0.32"}
{"level":"info","ts":"2026-07-13T16:02:29.823Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"b1411084-ce07-415f-a6d6-bfa697b781a6","rayCluster":"fail-76mdf","podIPs":["10.133.0.30","10.134.0.32"]}
{"level":"info","ts":"2026-07-13T16:02:32.597Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"bf76de2c-c90e-47bd-84d7-3ac5386ea91f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:32.597Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"bf76de2c-c90e-47bd-84d7-3ac5386ea91f","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:32.597Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"bf76de2c-c90e-47bd-84d7-3ac5386ea91f","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:32.597Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"bf76de2c-c90e-47bd-84d7-3ac5386ea91f"}
{"level":"info","ts":"2026-07-13T16:02:32.597Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"bf76de2c-c90e-47bd-84d7-3ac5386ea91f","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:02:34.091Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"72b4bbdb-f597-49d3-b7cb-e57d06a176a4"}
{"level":"info","ts":"2026-07-13T16:02:34.091Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"72b4bbdb-f597-49d3-b7cb-e57d06a176a4"}
{"level":"info","ts":"2026-07-13T16:02:34.091Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"72b4bbdb-f597-49d3-b7cb-e57d06a176a4","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:02:34.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"72b4bbdb-f597-49d3-b7cb-e57d06a176a4","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:02:34.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"72b4bbdb-f597-49d3-b7cb-e57d06a176a4","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:02:34.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"72b4bbdb-f597-49d3-b7cb-e57d06a176a4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:02:34.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"72b4bbdb-f597-49d3-b7cb-e57d06a176a4","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:02:34.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"72b4bbdb-f597-49d3-b7cb-e57d06a176a4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:02:34.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"72b4bbdb-f597-49d3-b7cb-e57d06a176a4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:02:34.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"72b4bbdb-f597-49d3-b7cb-e57d06a176a4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:02:34.091Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"72b4bbdb-f597-49d3-b7cb-e57d06a176a4","podName":"fail-76mdf-small-group-worker-xzxgm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:02:34.091Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"72b4bbdb-f597-49d3-b7cb-e57d06a176a4","podName":"fail-76mdf-small-group-worker-xzxgm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:02:34.091Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"72b4bbdb-f597-49d3-b7cb-e57d06a176a4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:02:34.091Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"72b4bbdb-f597-49d3-b7cb-e57d06a176a4","seconds":300}
{"level":"info","ts":"2026-07-13T16:02:34.944Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"b079a4de-5894-4f40-a4b3-a9d604846bfd","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:02:34.944Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"b079a4de-5894-4f40-a4b3-a9d604846bfd","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T16:02:34.944Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"b079a4de-5894-4f40-a4b3-a9d604846bfd","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T16:02:34.944Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"b079a4de-5894-4f40-a4b3-a9d604846bfd","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T16:02:35.117Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c8318078-0ec1-4a3b-9b3d-612ecb929a26"}
{"level":"info","ts":"2026-07-13T16:02:35.117Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c8318078-0ec1-4a3b-9b3d-612ecb929a26"}
{"level":"info","ts":"2026-07-13T16:02:35.117Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c8318078-0ec1-4a3b-9b3d-612ecb929a26","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:02:35.117Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c8318078-0ec1-4a3b-9b3d-612ecb929a26","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:02:35.117Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c8318078-0ec1-4a3b-9b3d-612ecb929a26","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:02:35.117Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c8318078-0ec1-4a3b-9b3d-612ecb929a26","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:02:35.117Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c8318078-0ec1-4a3b-9b3d-612ecb929a26","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:02:35.117Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c8318078-0ec1-4a3b-9b3d-612ecb929a26","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:02:35.117Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c8318078-0ec1-4a3b-9b3d-612ecb929a26","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:02:35.117Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c8318078-0ec1-4a3b-9b3d-612ecb929a26","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:02:35.117Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c8318078-0ec1-4a3b-9b3d-612ecb929a26","podName":"fail-76mdf-small-group-worker-xzxgm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:01:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:02:35.117Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c8318078-0ec1-4a3b-9b3d-612ecb929a26","podName":"fail-76mdf-small-group-worker-xzxgm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:01:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:02:35.117Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c8318078-0ec1-4a3b-9b3d-612ecb929a26","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T16:02:35.117Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"c8318078-0ec1-4a3b-9b3d-612ecb929a26","name":"fail-76mdf","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:01:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:02:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:02:35.131Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"72763a5f-c684-4b11-9999-b227b4b10869","rayCluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:35.131Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"18c2d238-f578-47c8-9b72-36d9b7a4b016","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:35.131Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7fa5e084-4280-4433-a8a8-f0a7136eab2d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:35.131Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7fa5e084-4280-4433-a8a8-f0a7136eab2d","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:35.131Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7fa5e084-4280-4433-a8a8-f0a7136eab2d","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:35.131Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"72763a5f-c684-4b11-9999-b227b4b10869","pod":"fail-76mdf-head-rbw2b","ip":"10.133.0.30"}
{"level":"info","ts":"2026-07-13T16:02:35.131Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"72763a5f-c684-4b11-9999-b227b4b10869","pod":"fail-76mdf-small-group-worker-xzxgm","ip":"10.134.0.32"}
{"level":"info","ts":"2026-07-13T16:02:35.131Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"72763a5f-c684-4b11-9999-b227b4b10869","rayCluster":"fail-76mdf","podIPs":["10.133.0.30","10.134.0.32"]}
{"level":"info","ts":"2026-07-13T16:02:35.131Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7fa5e084-4280-4433-a8a8-f0a7136eab2d"}
{"level":"info","ts":"2026-07-13T16:02:35.131Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7fa5e084-4280-4433-a8a8-f0a7136eab2d","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:02:35.147Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"18c2d238-f578-47c8-9b72-36d9b7a4b016","name":"fail-76mdf-head"}
{"level":"info","ts":"2026-07-13T16:02:35.158Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"18c2d238-f578-47c8-9b72-36d9b7a4b016","name":"fail-76mdf-workers"}
{"level":"info","ts":"2026-07-13T16:02:35.158Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"18c2d238-f578-47c8-9b72-36d9b7a4b016","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:35.597Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"b47afe7b-74d6-40cf-87ec-522aac32f9a5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:35.597Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"b47afe7b-74d6-40cf-87ec-522aac32f9a5","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:35.597Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"b47afe7b-74d6-40cf-87ec-522aac32f9a5","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:35.598Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"b47afe7b-74d6-40cf-87ec-522aac32f9a5"}
{"level":"info","ts":"2026-07-13T16:02:35.598Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"b47afe7b-74d6-40cf-87ec-522aac32f9a5","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:02:37.131Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"eb18134d-08a5-4938-bf61-7d8353684533"}
{"level":"info","ts":"2026-07-13T16:02:37.131Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"eb18134d-08a5-4938-bf61-7d8353684533"}
{"level":"info","ts":"2026-07-13T16:02:37.131Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"eb18134d-08a5-4938-bf61-7d8353684533","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:02:37.131Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"eb18134d-08a5-4938-bf61-7d8353684533","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:02:37.131Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"eb18134d-08a5-4938-bf61-7d8353684533","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:02:37.131Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"eb18134d-08a5-4938-bf61-7d8353684533","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:02:37.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"eb18134d-08a5-4938-bf61-7d8353684533","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:02:37.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"eb18134d-08a5-4938-bf61-7d8353684533","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:02:37.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"eb18134d-08a5-4938-bf61-7d8353684533","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:02:37.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"eb18134d-08a5-4938-bf61-7d8353684533","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:02:37.132Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"eb18134d-08a5-4938-bf61-7d8353684533","podName":"fail-76mdf-small-group-worker-xzxgm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:01:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:02:37.132Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"eb18134d-08a5-4938-bf61-7d8353684533","podName":"fail-76mdf-small-group-worker-xzxgm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:01:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:02:37.132Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"eb18134d-08a5-4938-bf61-7d8353684533","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:02:37.132Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"eb18134d-08a5-4938-bf61-7d8353684533","seconds":300}
{"level":"info","ts":"2026-07-13T16:02:38.599Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3b210577-3701-4c95-958a-c82ef93f3c99","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:38.599Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3b210577-3701-4c95-958a-c82ef93f3c99","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:38.599Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3b210577-3701-4c95-958a-c82ef93f3c99","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:38.599Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3b210577-3701-4c95-958a-c82ef93f3c99"}
{"level":"info","ts":"2026-07-13T16:02:38.599Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3b210577-3701-4c95-958a-c82ef93f3c99","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:02:41.599Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"6d2ac224-297c-418f-b23a-42c918610571","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:41.599Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"6d2ac224-297c-418f-b23a-42c918610571","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:41.599Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"6d2ac224-297c-418f-b23a-42c918610571","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:41.600Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"6d2ac224-297c-418f-b23a-42c918610571"}
{"level":"info","ts":"2026-07-13T16:02:41.600Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"6d2ac224-297c-418f-b23a-42c918610571","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:02:44.601Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"e4888f59-df43-4350-ae23-5a33b736e62b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:44.601Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"e4888f59-df43-4350-ae23-5a33b736e62b","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:44.601Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"e4888f59-df43-4350-ae23-5a33b736e62b","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:44.601Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"e4888f59-df43-4350-ae23-5a33b736e62b"}
{"level":"info","ts":"2026-07-13T16:02:44.601Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"e4888f59-df43-4350-ae23-5a33b736e62b","RayCluster":"fail-76mdf","State":""}
{"level":"info","ts":"2026-07-13T16:02:46.123Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"98ef9932-f6ae-41fd-81e1-d5d2ad559292"}
{"level":"info","ts":"2026-07-13T16:02:46.123Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"98ef9932-f6ae-41fd-81e1-d5d2ad559292"}
{"level":"info","ts":"2026-07-13T16:02:46.123Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"98ef9932-f6ae-41fd-81e1-d5d2ad559292","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:02:46.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"98ef9932-f6ae-41fd-81e1-d5d2ad559292","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:02:46.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"98ef9932-f6ae-41fd-81e1-d5d2ad559292","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:02:46.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"98ef9932-f6ae-41fd-81e1-d5d2ad559292","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:02:46.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"98ef9932-f6ae-41fd-81e1-d5d2ad559292","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:02:46.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"98ef9932-f6ae-41fd-81e1-d5d2ad559292","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:02:46.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"98ef9932-f6ae-41fd-81e1-d5d2ad559292","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:02:46.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"98ef9932-f6ae-41fd-81e1-d5d2ad559292","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:02:46.124Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"98ef9932-f6ae-41fd-81e1-d5d2ad559292","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-13T16:02:46.124Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"98ef9932-f6ae-41fd-81e1-d5d2ad559292","name":"fail-76mdf","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:02:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:02:46Z","stateTransitionTimes":{"ready":"2026-07-13T16:02:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:02:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:02:46.136Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"b6d1e912-3c1c-4201-894e-0b16aeb2bad3","rayCluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:46.136Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"5b1c8636-32d3-4030-8b77-e335987005d3","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:46.136Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cb0f82b0-321c-4cf3-a3b4-e093944ab19c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:46.136Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cb0f82b0-321c-4cf3-a3b4-e093944ab19c","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:46.136Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cb0f82b0-321c-4cf3-a3b4-e093944ab19c","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:46.136Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"b6d1e912-3c1c-4201-894e-0b16aeb2bad3","pod":"fail-76mdf-head-rbw2b","ip":"10.133.0.30"}
{"level":"info","ts":"2026-07-13T16:02:46.136Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"b6d1e912-3c1c-4201-894e-0b16aeb2bad3","pod":"fail-76mdf-small-group-worker-xzxgm","ip":"10.134.0.32"}
{"level":"info","ts":"2026-07-13T16:02:46.136Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"b6d1e912-3c1c-4201-894e-0b16aeb2bad3","rayCluster":"fail-76mdf","podIPs":["10.133.0.30","10.134.0.32"]}
{"level":"info","ts":"2026-07-13T16:02:46.136Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cb0f82b0-321c-4cf3-a3b4-e093944ab19c"}
{"level":"info","ts":"2026-07-13T16:02:46.136Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cb0f82b0-321c-4cf3-a3b4-e093944ab19c","head service name":"fail-76mdf-head-svc","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:02:46.136Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cb0f82b0-321c-4cf3-a3b4-e093944ab19c","head service URL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-13T16:02:46.136Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cb0f82b0-321c-4cf3-a3b4-e093944ab19c"}
{"level":"info","ts":"2026-07-13T16:02:46.136Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cb0f82b0-321c-4cf3-a3b4-e093944ab19c","command":["if","!","ray","job","status","--address","http://fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","fail-jfd4l",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","--no-wait","--submission-id","fail-jfd4l","--","python /home/ray/jobs/fail.py",";","fi",";","ray","job","logs","--address","http://fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","--follow","fail-jfd4l"]}
{"level":"info","ts":"2026-07-13T16:02:46.149Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cb0f82b0-321c-4cf3-a3b4-e093944ab19c","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-07-13T16:02:46.149Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cb0f82b0-321c-4cf3-a3b4-e093944ab19c","SubmissionMode":"K8sJobMode","RayCluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:46.149Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cb0f82b0-321c-4cf3-a3b4-e093944ab19c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","dashboardURL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T16:02:46.149Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cb0f82b0-321c-4cf3-a3b4-e093944ab19c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T16:02:46.153Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"5b1c8636-32d3-4030-8b77-e335987005d3","name":"fail-76mdf-head"}
{"level":"info","ts":"2026-07-13T16:02:46.159Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7000eb41-9c98-4334-b837-73e7804b8579","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:46.159Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7000eb41-9c98-4334-b837-73e7804b8579","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:46.159Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7000eb41-9c98-4334-b837-73e7804b8579","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:46.159Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7000eb41-9c98-4334-b837-73e7804b8579"}
{"level":"info","ts":"2026-07-13T16:02:46.160Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7000eb41-9c98-4334-b837-73e7804b8579","head service name":"fail-76mdf-head-svc","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:02:46.160Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7000eb41-9c98-4334-b837-73e7804b8579","head service URL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-13T16:02:46.160Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7000eb41-9c98-4334-b837-73e7804b8579","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-07-13T16:02:46.160Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7000eb41-9c98-4334-b837-73e7804b8579","SubmissionMode":"K8sJobMode","RayCluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:46.160Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7000eb41-9c98-4334-b837-73e7804b8579","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","dashboardURL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T16:02:46.160Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7000eb41-9c98-4334-b837-73e7804b8579","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T16:02:46.166Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7000eb41-9c98-4334-b837-73e7804b8579","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-13T16:02:46.166Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7000eb41-9c98-4334-b837-73e7804b8579"}
{"level":"error","ts":"2026-07-13T16:02:46.166Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"7000eb41-9c98-4334-b837-73e7804b8579","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:02:46.166Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"fb6bf229-558b-4bd7-a0dd-08611126dc40","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:46.166Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"fb6bf229-558b-4bd7-a0dd-08611126dc40","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:46.166Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"fb6bf229-558b-4bd7-a0dd-08611126dc40","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:46.166Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"fb6bf229-558b-4bd7-a0dd-08611126dc40"}
{"level":"info","ts":"2026-07-13T16:02:46.167Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"5b1c8636-32d3-4030-8b77-e335987005d3","name":"fail-76mdf-workers"}
{"level":"info","ts":"2026-07-13T16:02:46.167Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"5b1c8636-32d3-4030-8b77-e335987005d3","cluster":"fail-76mdf"}
{"level":"error","ts":"2026-07-13T16:02:46.226Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"fb6bf229-558b-4bd7-a0dd-08611126dc40","JobId":"fail-jfd4l","error":"Job fail-jfd4l does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:02:46.226Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"fb6bf229-558b-4bd7-a0dd-08611126dc40"}
{"level":"error","ts":"2026-07-13T16:02:46.226Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"fb6bf229-558b-4bd7-a0dd-08611126dc40","error":"Job fail-jfd4l does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:02:46.226Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"aa44c665-e6f4-48a9-b32c-0cc6bdf83673","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:46.226Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"aa44c665-e6f4-48a9-b32c-0cc6bdf83673","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:46.226Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"aa44c665-e6f4-48a9-b32c-0cc6bdf83673","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:46.226Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"aa44c665-e6f4-48a9-b32c-0cc6bdf83673"}
{"level":"error","ts":"2026-07-13T16:02:46.230Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"aa44c665-e6f4-48a9-b32c-0cc6bdf83673","JobId":"fail-jfd4l","error":"Job fail-jfd4l does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:02:46.230Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"aa44c665-e6f4-48a9-b32c-0cc6bdf83673"}
{"level":"error","ts":"2026-07-13T16:02:46.230Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"aa44c665-e6f4-48a9-b32c-0cc6bdf83673","error":"Job fail-jfd4l does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:02:46.236Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"42eefd3e-6002-4718-8a25-d179d3b19d9a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:46.236Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"42eefd3e-6002-4718-8a25-d179d3b19d9a","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:46.236Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"42eefd3e-6002-4718-8a25-d179d3b19d9a","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:46.236Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"42eefd3e-6002-4718-8a25-d179d3b19d9a"}
{"level":"error","ts":"2026-07-13T16:02:46.240Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"42eefd3e-6002-4718-8a25-d179d3b19d9a","JobId":"fail-jfd4l","error":"Job fail-jfd4l does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:02:46.240Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"42eefd3e-6002-4718-8a25-d179d3b19d9a"}
{"level":"error","ts":"2026-07-13T16:02:46.240Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"42eefd3e-6002-4718-8a25-d179d3b19d9a","error":"Job fail-jfd4l does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:02:46.281Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d4e0832a-0934-460e-8dc5-ea431fcee1e0","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:46.281Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d4e0832a-0934-460e-8dc5-ea431fcee1e0","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:46.281Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d4e0832a-0934-460e-8dc5-ea431fcee1e0","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:46.281Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d4e0832a-0934-460e-8dc5-ea431fcee1e0"}
{"level":"error","ts":"2026-07-13T16:02:46.285Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d4e0832a-0934-460e-8dc5-ea431fcee1e0","JobId":"fail-jfd4l","error":"Job fail-jfd4l does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:02:46.285Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d4e0832a-0934-460e-8dc5-ea431fcee1e0"}
{"level":"error","ts":"2026-07-13T16:02:46.285Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d4e0832a-0934-460e-8dc5-ea431fcee1e0","error":"Job fail-jfd4l does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:02:46.365Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"1987a6eb-a292-4043-90ed-775c3ce19740","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:46.365Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"1987a6eb-a292-4043-90ed-775c3ce19740","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:46.365Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"1987a6eb-a292-4043-90ed-775c3ce19740","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:46.365Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"1987a6eb-a292-4043-90ed-775c3ce19740"}
{"level":"error","ts":"2026-07-13T16:02:46.369Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"1987a6eb-a292-4043-90ed-775c3ce19740","JobId":"fail-jfd4l","error":"Job fail-jfd4l does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:02:46.369Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"1987a6eb-a292-4043-90ed-775c3ce19740"}
{"level":"error","ts":"2026-07-13T16:02:46.369Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"1987a6eb-a292-4043-90ed-775c3ce19740","error":"Job fail-jfd4l does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:02:46.530Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d2165a53-87b0-4f34-9662-faa934e29944","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:46.530Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d2165a53-87b0-4f34-9662-faa934e29944","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:46.530Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d2165a53-87b0-4f34-9662-faa934e29944","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:46.530Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d2165a53-87b0-4f34-9662-faa934e29944"}
{"level":"error","ts":"2026-07-13T16:02:46.537Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d2165a53-87b0-4f34-9662-faa934e29944","JobId":"fail-jfd4l","error":"Job fail-jfd4l does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:02:46.537Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d2165a53-87b0-4f34-9662-faa934e29944"}
{"level":"error","ts":"2026-07-13T16:02:46.537Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d2165a53-87b0-4f34-9662-faa934e29944","error":"Job fail-jfd4l does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:02:46.858Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d0011293-077c-45b8-99d9-6a219566c6c3","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:46.858Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d0011293-077c-45b8-99d9-6a219566c6c3","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:46.858Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d0011293-077c-45b8-99d9-6a219566c6c3","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:46.858Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d0011293-077c-45b8-99d9-6a219566c6c3"}
{"level":"error","ts":"2026-07-13T16:02:46.863Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d0011293-077c-45b8-99d9-6a219566c6c3","JobId":"fail-jfd4l","error":"Job fail-jfd4l does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:02:46.863Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d0011293-077c-45b8-99d9-6a219566c6c3"}
{"level":"error","ts":"2026-07-13T16:02:46.863Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"d0011293-077c-45b8-99d9-6a219566c6c3","error":"Job fail-jfd4l does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:02:47.171Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"c21101f8-b0f7-4837-be98-601c40ef2a2b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:47.171Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"c21101f8-b0f7-4837-be98-601c40ef2a2b","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:47.171Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"c21101f8-b0f7-4837-be98-601c40ef2a2b","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:47.171Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"c21101f8-b0f7-4837-be98-601c40ef2a2b"}
{"level":"error","ts":"2026-07-13T16:02:47.175Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"c21101f8-b0f7-4837-be98-601c40ef2a2b","JobId":"fail-jfd4l","error":"Job fail-jfd4l does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:02:47.175Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"c21101f8-b0f7-4837-be98-601c40ef2a2b"}
{"level":"error","ts":"2026-07-13T16:02:47.175Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"c21101f8-b0f7-4837-be98-601c40ef2a2b","error":"Job fail-jfd4l does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:02:47.503Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"f656d0f2-ba05-4753-bae7-59ec3442802b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:47.503Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"f656d0f2-ba05-4753-bae7-59ec3442802b","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:47.503Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"f656d0f2-ba05-4753-bae7-59ec3442802b","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:47.503Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"f656d0f2-ba05-4753-bae7-59ec3442802b"}
{"level":"error","ts":"2026-07-13T16:02:47.508Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"f656d0f2-ba05-4753-bae7-59ec3442802b","JobId":"fail-jfd4l","error":"Job fail-jfd4l does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:02:47.508Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"f656d0f2-ba05-4753-bae7-59ec3442802b"}
{"level":"error","ts":"2026-07-13T16:02:47.508Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"f656d0f2-ba05-4753-bae7-59ec3442802b","error":"Job fail-jfd4l does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:02:48.136Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a56c8c8e-675a-46c8-bc03-6d81c70e8fd1"}
{"level":"info","ts":"2026-07-13T16:02:48.136Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a56c8c8e-675a-46c8-bc03-6d81c70e8fd1"}
{"level":"info","ts":"2026-07-13T16:02:48.136Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a56c8c8e-675a-46c8-bc03-6d81c70e8fd1","1 head service found":"fail-76mdf-head-svc"}
{"level":"info","ts":"2026-07-13T16:02:48.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a56c8c8e-675a-46c8-bc03-6d81c70e8fd1","Found 1 head Pod":"fail-76mdf-head-rbw2b","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:02:48.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a56c8c8e-675a-46c8-bc03-6d81c70e8fd1","head Pod":"fail-76mdf-head-rbw2b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-76mdf-head-rbw2b. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:02:48.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a56c8c8e-675a-46c8-bc03-6d81c70e8fd1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:02:48.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a56c8c8e-675a-46c8-bc03-6d81c70e8fd1","worker Pod":"fail-76mdf-small-group-worker-xzxgm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-76mdf-small-group-worker-xzxgm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:02:48.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a56c8c8e-675a-46c8-bc03-6d81c70e8fd1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:02:48.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a56c8c8e-675a-46c8-bc03-6d81c70e8fd1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:02:48.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a56c8c8e-675a-46c8-bc03-6d81c70e8fd1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:02:48.137Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a56c8c8e-675a-46c8-bc03-6d81c70e8fd1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:02:48.137Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"a56c8c8e-675a-46c8-bc03-6d81c70e8fd1","seconds":300}
{"level":"info","ts":"2026-07-13T16:02:50.069Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"4f9f7924-27e5-424e-a1e5-2613867edd43","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:50.069Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"4f9f7924-27e5-424e-a1e5-2613867edd43","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:50.069Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"4f9f7924-27e5-424e-a1e5-2613867edd43","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:50.069Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"4f9f7924-27e5-424e-a1e5-2613867edd43"}
{"level":"error","ts":"2026-07-13T16:02:50.074Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"4f9f7924-27e5-424e-a1e5-2613867edd43","JobId":"fail-jfd4l","error":"Job fail-jfd4l does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:02:50.074Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"4f9f7924-27e5-424e-a1e5-2613867edd43"}
{"level":"error","ts":"2026-07-13T16:02:50.074Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"4f9f7924-27e5-424e-a1e5-2613867edd43","error":"Job fail-jfd4l does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:02:55.195Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"491be046-8e17-460a-ace6-6584a84222d0","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:55.195Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"491be046-8e17-460a-ace6-6584a84222d0","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:55.195Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"491be046-8e17-460a-ace6-6584a84222d0","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:55.195Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"491be046-8e17-460a-ace6-6584a84222d0"}
{"level":"info","ts":"2026-07-13T16:02:55.201Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"491be046-8e17-460a-ace6-6584a84222d0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","dashboardURL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:02:53Z"},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","dashboardURL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:02:46Z","stateTransitionTimes":{"ready":"2026-07-13T16:02:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:02:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:02:55.201Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"491be046-8e17-460a-ace6-6584a84222d0","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T16:02:55.214Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"9b25729f-e961-4a39-ac8c-475b71ee847d","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:55.214Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"9b25729f-e961-4a39-ac8c-475b71ee847d","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:55.214Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"9b25729f-e961-4a39-ac8c-475b71ee847d","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:55.214Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"9b25729f-e961-4a39-ac8c-475b71ee847d"}
{"level":"info","ts":"2026-07-13T16:02:55.216Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"9b25729f-e961-4a39-ac8c-475b71ee847d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:02:53Z"},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","dashboardURL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:02:46Z","stateTransitionTimes":{"ready":"2026-07-13T16:02:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:02:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:02:53Z"},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","dashboardURL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:02:46Z","stateTransitionTimes":{"ready":"2026-07-13T16:02:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:02:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:02:58.214Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"a44f939e-b0a4-4450-851e-e177d114d3a2","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:58.214Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"a44f939e-b0a4-4450-851e-e177d114d3a2","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:58.214Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"a44f939e-b0a4-4450-851e-e177d114d3a2","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:58.214Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"a44f939e-b0a4-4450-851e-e177d114d3a2"}
{"level":"info","ts":"2026-07-13T16:02:58.217Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"a44f939e-b0a4-4450-851e-e177d114d3a2","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:02:53Z"},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","dashboardURL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:02:46Z","stateTransitionTimes":{"ready":"2026-07-13T16:02:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:02:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:02:53Z","endTime":"2026-07-13T16:02:55Z"},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","dashboardURL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:02:53,918\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-jfd4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:02:46Z","stateTransitionTimes":{"ready":"2026-07-13T16:02:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:02:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:02:58.217Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"a44f939e-b0a4-4450-851e-e177d114d3a2","old JobStatus":"PENDING","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T16:02:58.232Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"9a2996f4-ef0c-40b9-bcae-d753e1be9e3d","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:02:58.232Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"9a2996f4-ef0c-40b9-bcae-d753e1be9e3d","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:02:58.232Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"9a2996f4-ef0c-40b9-bcae-d753e1be9e3d","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:02:58.233Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"9a2996f4-ef0c-40b9-bcae-d753e1be9e3d"}
{"level":"info","ts":"2026-07-13T16:02:58.235Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"9a2996f4-ef0c-40b9-bcae-d753e1be9e3d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:02:53Z","endTime":"2026-07-13T16:02:55Z"},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","dashboardURL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:02:53,918\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-jfd4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:02:46Z","stateTransitionTimes":{"ready":"2026-07-13T16:02:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:02:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:02:53Z","endTime":"2026-07-13T16:02:55Z"},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","dashboardURL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:02:53,918\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-jfd4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:02:46Z","stateTransitionTimes":{"ready":"2026-07-13T16:02:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:02:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:03:01.233Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"6d166a0f-8e20-4f9b-8afa-4fdbb5996576","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:01.233Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"6d166a0f-8e20-4f9b-8afa-4fdbb5996576","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:03:01.233Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"6d166a0f-8e20-4f9b-8afa-4fdbb5996576","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:01.233Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"6d166a0f-8e20-4f9b-8afa-4fdbb5996576"}
{"level":"info","ts":"2026-07-13T16:03:01.236Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"6d166a0f-8e20-4f9b-8afa-4fdbb5996576","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:02:53Z","endTime":"2026-07-13T16:02:55Z"},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","dashboardURL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:02:53,918\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-jfd4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:02:46Z","stateTransitionTimes":{"ready":"2026-07-13T16:02:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:02:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:02:53Z","endTime":"2026-07-13T16:02:55Z"},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","dashboardURL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:02:53,918\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-jfd4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:02:46Z","stateTransitionTimes":{"ready":"2026-07-13T16:02:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:02:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:03:04.237Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"b4062eef-affc-4d0b-a4f9-04ae0a3f241f","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:04.237Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"b4062eef-affc-4d0b-a4f9-04ae0a3f241f","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:03:04.237Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"b4062eef-affc-4d0b-a4f9-04ae0a3f241f","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:04.237Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"b4062eef-affc-4d0b-a4f9-04ae0a3f241f"}
{"level":"info","ts":"2026-07-13T16:03:04.240Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"b4062eef-affc-4d0b-a4f9-04ae0a3f241f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:02:53Z","endTime":"2026-07-13T16:02:55Z"},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","dashboardURL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:02:53,918\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-jfd4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:02:46Z","stateTransitionTimes":{"ready":"2026-07-13T16:02:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:02:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:02:53Z","endTime":"2026-07-13T16:02:55Z"},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","dashboardURL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:02:53,918\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-jfd4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:02:46Z","stateTransitionTimes":{"ready":"2026-07-13T16:02:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:02:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:03:04.928Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cfbe66db-918b-4e0d-af32-beaa43f46b8e","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:04.928Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cfbe66db-918b-4e0d-af32-beaa43f46b8e","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:03:04.928Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cfbe66db-918b-4e0d-af32-beaa43f46b8e","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:04.928Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cfbe66db-918b-4e0d-af32-beaa43f46b8e"}
{"level":"info","ts":"2026-07-13T16:03:04.931Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"cfbe66db-918b-4e0d-af32-beaa43f46b8e","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:02:53Z","endTime":"2026-07-13T16:02:55Z"},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","dashboardURL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:02:53,918\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-jfd4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:02:46Z","stateTransitionTimes":{"ready":"2026-07-13T16:02:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:02:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:02:53Z","endTime":"2026-07-13T16:02:55Z"},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","dashboardURL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:02:53,918\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-jfd4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:02:46Z","stateTransitionTimes":{"ready":"2026-07-13T16:02:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:02:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:03:05.935Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"9c343e7e-a28e-401d-bf78-54386d4b5409","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:05.935Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"9c343e7e-a28e-401d-bf78-54386d4b5409","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:03:05.935Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"9c343e7e-a28e-401d-bf78-54386d4b5409","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:05.936Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"9c343e7e-a28e-401d-bf78-54386d4b5409"}
{"level":"info","ts":"2026-07-13T16:03:05.938Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"9c343e7e-a28e-401d-bf78-54386d4b5409","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:02:53Z","endTime":"2026-07-13T16:02:55Z"},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","dashboardURL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:02:53,918\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-jfd4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:02:46Z","stateTransitionTimes":{"ready":"2026-07-13T16:02:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:02:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:02:53Z","endTime":"2026-07-13T16:02:55Z"},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","dashboardURL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:02:53,918\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-jfd4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:02:46Z","stateTransitionTimes":{"ready":"2026-07-13T16:02:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:02:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:03:05.950Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"dbc29b87-cbe1-4344-907e-822e10cfe0e2","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:05.950Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"dbc29b87-cbe1-4344-907e-822e10cfe0e2","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:03:05.950Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"dbc29b87-cbe1-4344-907e-822e10cfe0e2","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:05.951Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"dbc29b87-cbe1-4344-907e-822e10cfe0e2"}
{"level":"info","ts":"2026-07-13T16:03:05.953Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"dbc29b87-cbe1-4344-907e-822e10cfe0e2","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:02:53Z","endTime":"2026-07-13T16:02:55Z"},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","dashboardURL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:02:53,918\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-jfd4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:02:46Z","stateTransitionTimes":{"ready":"2026-07-13T16:02:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:02:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:02:53Z","endTime":"2026-07-13T16:02:55Z"},"jobId":"fail-jfd4l","rayClusterName":"fail-76mdf","dashboardURL":"fail-76mdf-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Failed","reason":"AppFailed","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:02:53,918\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-jfd4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:00:59Z","succeeded":0,"failed":1,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:02:46Z","stateTransitionTimes":{"ready":"2026-07-13T16:02:46Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.30","serviceIP":"10.133.0.30","podName":"fail-76mdf-head-rbw2b","serviceName":"fail-76mdf-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:01:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:02:46Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:00:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:00:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:03:05.953Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"dbc29b87-cbe1-4344-907e-822e10cfe0e2","old JobStatus":"FAILED","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-07-13T16:03:06.021Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"8f833495-b4d4-4efc-9424-16541c3210b9","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:06.021Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"8f833495-b4d4-4efc-9424-16541c3210b9","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 16:03:05 +0000 UTC","Now":"2026-07-13T16:03:06.021Z","ShutdownTime":"2026-07-13T16:03:05.000Z"}
{"level":"info","ts":"2026-07-13T16:03:06.229Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"12bcc511-2651-45ac-95c1-c9af1b36096e","DeletionTimestamp":"2026-07-13 16:03:06 +0000 UTC"}
{"level":"info","ts":"2026-07-13T16:03:06.229Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"12bcc511-2651-45ac-95c1-c9af1b36096e","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T16:03:06.244Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"fdffe7da-2e75-4283-9e91-f22c6b232336"}
{"level":"info","ts":"2026-07-13T16:03:06.253Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"a543286e-dc10-436c-9f23-3004a16781a8"}
{"level":"info","ts":"2026-07-13T16:03:06.260Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"7510655b-16a9-47c4-86be-e54ecb444028","namespacedName":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:06.260Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"3462d50b-2bad-47f8-83ab-18ef6cd41ed9"}
{"level":"info","ts":"2026-07-13T16:03:06.260Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"7510655b-16a9-47c4-86be-e54ecb444028","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:03:06.260Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"reconcileID":"4ce34ceb-735c-4dc9-80af-bcfeef723054"}
{"level":"info","ts":"2026-07-13T16:03:06.260Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"7510655b-16a9-47c4-86be-e54ecb444028","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:03:06.260Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"7510655b-16a9-47c4-86be-e54ecb444028","namespace":"test-ns-x5qvg","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-13T16:03:06.260Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"711c5100-0167-43a9-a3b0-f4ef8f9348d3"}
{"level":"info","ts":"2026-07-13T16:03:06.260Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"2630db57-98cc-496c-92f4-40e96acd1ffb","rayCluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:03:06.260Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"2630db57-98cc-496c-92f4-40e96acd1ffb","pod":"fail-76mdf-head-rbw2b","ip":"10.133.0.30"}
{"level":"info","ts":"2026-07-13T16:03:06.260Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"2630db57-98cc-496c-92f4-40e96acd1ffb","pod":"fail-76mdf-small-group-worker-xzxgm","ip":"10.134.0.32"}
{"level":"info","ts":"2026-07-13T16:03:06.260Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"2630db57-98cc-496c-92f4-40e96acd1ffb","rayCluster":"fail-76mdf","podIPs":["10.133.0.30","10.134.0.32"]}
{"level":"info","ts":"2026-07-13T16:03:06.266Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"7510655b-16a9-47c4-86be-e54ecb444028","name":"kuberay-gateway-access","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:03:06.270Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"7510655b-16a9-47c4-86be-e54ecb444028","name":"test-ns-x5qvg-fail-76mdf","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-13T16:03:06.275Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"7510655b-16a9-47c4-86be-e54ecb444028","configMap":"kube-rbac-proxy-config-fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:03:06.280Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"7510655b-16a9-47c4-86be-e54ecb444028","serviceAccount":"fail-76mdf-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:03:06.325Z","logger":"raycluster-resource","msg":"default","name":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:03:06.326Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-76mdf","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:03:06.335Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"7510655b-16a9-47c4-86be-e54ecb444028","cluster":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:03:06.335Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"692eebe4-ad0d-490a-a059-cad3b12031f8","namespacedName":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:06.335Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"692eebe4-ad0d-490a-a059-cad3b12031f8"}
{"level":"info","ts":"2026-07-13T16:03:06.335Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"692eebe4-ad0d-490a-a059-cad3b12031f8","namespace":"test-ns-x5qvg","authClusters":0}
{"level":"info","ts":"2026-07-13T16:03:06.335Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"692eebe4-ad0d-490a-a059-cad3b12031f8","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:03:06.335Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"bd500f41-794a-488c-a86a-518de516ecc1","namespace":"test-ns-x5qvg","clusterName":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:03:06.335Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"02933beb-0cab-4b87-b03a-9f51f18d7fce","namespacedName":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:06.335Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"02933beb-0cab-4b87-b03a-9f51f18d7fce"}
{"level":"info","ts":"2026-07-13T16:03:06.335Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"02933beb-0cab-4b87-b03a-9f51f18d7fce","namespace":"test-ns-x5qvg","authClusters":0}
{"level":"info","ts":"2026-07-13T16:03:06.335Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"02933beb-0cab-4b87-b03a-9f51f18d7fce","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:03:06.336Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3c4fe8d2-45a2-4b47-b92a-6340330bff63"}
{"level":"info","ts":"2026-07-13T16:03:06.335Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"272b939c-c576-4875-a900-beb642fa8f6d"}
{"level":"info","ts":"2026-07-13T16:03:06.341Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"e263c46c-212a-4672-9075-4ded1fbc8ef1"}
{"level":"info","ts":"2026-07-13T16:03:06.342Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"d8238271-fdf2-4749-8837-3f23ad0767ba","namespacedName":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:06.342Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"d8238271-fdf2-4749-8837-3f23ad0767ba"}
{"level":"info","ts":"2026-07-13T16:03:06.342Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"d8238271-fdf2-4749-8837-3f23ad0767ba","namespace":"test-ns-x5qvg","authClusters":0}
{"level":"info","ts":"2026-07-13T16:03:06.342Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"d8238271-fdf2-4749-8837-3f23ad0767ba","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:03:06.351Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"66bd339a-9892-4d71-9929-44a47dffd456"}
{"level":"error","ts":"2026-07-13T16:03:06.354Z","msg":"Failed to delete worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"bd500f41-794a-488c-a86a-518de516ecc1","certificate":"ray-worker-cert-fail-76mdf","error":"certificates.cert-manager.io \"ray-worker-cert-fail-76mdf\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:268\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:06.377Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"bd500f41-794a-488c-a86a-518de516ecc1","errorCount":1}
{"level":"info","ts":"2026-07-13T16:03:06.377Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"bd500f41-794a-488c-a86a-518de516ecc1"}
{"level":"error","ts":"2026-07-13T16:03:06.377Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"bd500f41-794a-488c-a86a-518de516ecc1","error":"certificates.cert-manager.io \"ray-worker-cert-fail-76mdf\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:06.383Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"b25eafc6-bc2c-4d9f-96d7-a1df53e1bbcf","namespace":"test-ns-x5qvg","clusterName":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:03:06.383Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-76mdf","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-76mdf","reconcileID":"b25eafc6-bc2c-4d9f-96d7-a1df53e1bbcf","namespace":"test-ns-x5qvg","clusterName":"fail-76mdf"}
{"level":"info","ts":"2026-07-13T16:03:07.241Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-x5qvg"},"reconcileID":"3519b670-0baa-4f12-81df-712cc8bee346"}
{"level":"info","ts":"2026-07-13T16:03:07.247Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c1d96999-3e93-47a2-a7e8-b88f364f3f80","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:07.247Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c1d96999-3e93-47a2-a7e8-b88f364f3f80","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T16:03:07.255Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c1d96999-3e93-47a2-a7e8-b88f364f3f80"}
{"level":"info","ts":"2026-07-13T16:03:07.255Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c1d96999-3e93-47a2-a7e8-b88f364f3f80","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-13T16:03:07.255Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c1d96999-3e93-47a2-a7e8-b88f364f3f80","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-qdx97","rayClusterName":"fail-k8s-job-vrx7k","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T16:03:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T16:03:07.255Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c1d96999-3e93-47a2-a7e8-b88f364f3f80","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T16:03:07.264Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"68115220-1e94-455d-84c2-3efd60d462d3","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:07.264Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"68115220-1e94-455d-84c2-3efd60d462d3"}
{"level":"info","ts":"2026-07-13T16:03:07.264Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"68115220-1e94-455d-84c2-3efd60d462d3","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-13T16:03:07.264Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"68115220-1e94-455d-84c2-3efd60d462d3","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-pgkbb","rayClusterName":"fail-k8s-job-rsslb","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T16:03:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T16:03:07.264Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"68115220-1e94-455d-84c2-3efd60d462d3","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T16:03:07.269Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"68115220-1e94-455d-84c2-3efd60d462d3","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail-k8s-job\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-13T16:03:07.269Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"68115220-1e94-455d-84c2-3efd60d462d3"}
{"level":"error","ts":"2026-07-13T16:03:07.269Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"68115220-1e94-455d-84c2-3efd60d462d3","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail-k8s-job\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:07.269Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"84e3e2d0-b19d-4f92-b550-3bf52df0ca45","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:07.269Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"84e3e2d0-b19d-4f92-b550-3bf52df0ca45","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.269Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"84e3e2d0-b19d-4f92-b550-3bf52df0ca45","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:07.269Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"84e3e2d0-b19d-4f92-b550-3bf52df0ca45","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:07.274Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.274Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:03:07.282Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"84e3e2d0-b19d-4f92-b550-3bf52df0ca45","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:07.283Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"84e3e2d0-b19d-4f92-b550-3bf52df0ca45"}
{"level":"info","ts":"2026-07-13T16:03:07.283Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"84e3e2d0-b19d-4f92-b550-3bf52df0ca45","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:07.283Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"86d381b7-2edf-4698-9bce-b5eafd624679","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:07.283Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"86d381b7-2edf-4698-9bce-b5eafd624679","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.283Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"86d381b7-2edf-4698-9bce-b5eafd624679","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:07.283Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ff72c6c5-a109-4827-ab41-1b0a0a6f9dda"}
{"level":"info","ts":"2026-07-13T16:03:07.283Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ff72c6c5-a109-4827-ab41-1b0a0a6f9dda"}
{"level":"info","ts":"2026-07-13T16:03:07.283Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"c1343773-34cc-4f0a-99ac-06d8e49403b1","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.283Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"da3240e9-be45-4de9-8f15-39cfeca12a2d","namespacedName":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:07.283Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"a2a6c8dc-9d12-41be-b2d8-2e8858cf9429","rayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.283Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"86d381b7-2edf-4698-9bce-b5eafd624679"}
{"level":"info","ts":"2026-07-13T16:03:07.283Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"86d381b7-2edf-4698-9bce-b5eafd624679","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:07.283Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"da3240e9-be45-4de9-8f15-39cfeca12a2d","mode":"IntegratedOAuth","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.283Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"a2a6c8dc-9d12-41be-b2d8-2e8858cf9429","rayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.283Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"da3240e9-be45-4de9-8f15-39cfeca12a2d","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.283Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"da3240e9-be45-4de9-8f15-39cfeca12a2d","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.283Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1583ea3f-6435-4b76-9ac4-5db655e5bcbc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:07.283Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1583ea3f-6435-4b76-9ac4-5db655e5bcbc","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.283Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1583ea3f-6435-4b76-9ac4-5db655e5bcbc","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:07.283Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1583ea3f-6435-4b76-9ac4-5db655e5bcbc"}
{"level":"info","ts":"2026-07-13T16:03:07.283Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1583ea3f-6435-4b76-9ac4-5db655e5bcbc","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:07.321Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"a2a6c8dc-9d12-41be-b2d8-2e8858cf9429","rayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.321Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"a2a6c8dc-9d12-41be-b2d8-2e8858cf9429","rayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.321Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.321Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:03:07.327Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ff72c6c5-a109-4827-ab41-1b0a0a6f9dda","name":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:07.327Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ff72c6c5-a109-4827-ab41-1b0a0a6f9dda","error":"head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found"}
{"level":"error","ts":"2026-07-13T16:03:07.327Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ff72c6c5-a109-4827-ab41-1b0a0a6f9dda","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:07.327Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ff72c6c5-a109-4827-ab41-1b0a0a6f9dda","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found"}
{"level":"info","ts":"2026-07-13T16:03:07.327Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ff72c6c5-a109-4827-ab41-1b0a0a6f9dda","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:07.327Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ff72c6c5-a109-4827-ab41-1b0a0a6f9dda","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:07.327Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ff72c6c5-a109-4827-ab41-1b0a0a6f9dda","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:07.327Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ff72c6c5-a109-4827-ab41-1b0a0a6f9dda","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T16:03:07.327Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ff72c6c5-a109-4827-ab41-1b0a0a6f9dda","name":"fail-k8s-job-vrx7k","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:03:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-vrx7k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:03:07.330Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"a2a6c8dc-9d12-41be-b2d8-2e8858cf9429","rayCluster":"fail-k8s-job-vrx7k","secret":"ca-secret"}
{"level":"info","ts":"2026-07-13T16:03:07.330Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"a2a6c8dc-9d12-41be-b2d8-2e8858cf9429","rayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.330Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"53bb4ca1-d5c3-46d6-98da-2343a88711bc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:07.330Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"53bb4ca1-d5c3-46d6-98da-2343a88711bc","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.330Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"53bb4ca1-d5c3-46d6-98da-2343a88711bc","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:07.331Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"53bb4ca1-d5c3-46d6-98da-2343a88711bc"}
{"level":"info","ts":"2026-07-13T16:03:07.331Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"53bb4ca1-d5c3-46d6-98da-2343a88711bc","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:07.333Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ff72c6c5-a109-4827-ab41-1b0a0a6f9dda","name":"fail-k8s-job-vrx7k","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-vrx7k\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k"}}
{"level":"info","ts":"2026-07-13T16:03:07.333Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ff72c6c5-a109-4827-ab41-1b0a0a6f9dda"}
{"level":"error","ts":"2026-07-13T16:03:07.333Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ff72c6c5-a109-4827-ab41-1b0a0a6f9dda","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:07.333Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d04c6d29-996b-42aa-9f1b-59c963c8920c"}
{"level":"info","ts":"2026-07-13T16:03:07.333Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d04c6d29-996b-42aa-9f1b-59c963c8920c"}
{"level":"info","ts":"2026-07-13T16:03:07.333Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d04c6d29-996b-42aa-9f1b-59c963c8920c","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:07.333Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d04c6d29-996b-42aa-9f1b-59c963c8920c","error":"head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found"}
{"level":"error","ts":"2026-07-13T16:03:07.333Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d04c6d29-996b-42aa-9f1b-59c963c8920c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:07.333Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d04c6d29-996b-42aa-9f1b-59c963c8920c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found"}
{"level":"info","ts":"2026-07-13T16:03:07.333Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d04c6d29-996b-42aa-9f1b-59c963c8920c","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:07.333Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d04c6d29-996b-42aa-9f1b-59c963c8920c","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:07.333Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d04c6d29-996b-42aa-9f1b-59c963c8920c","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:07.333Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d04c6d29-996b-42aa-9f1b-59c963c8920c","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T16:03:07.333Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d04c6d29-996b-42aa-9f1b-59c963c8920c","name":"fail-k8s-job-vrx7k","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:03:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-vrx7k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:03:07.334Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"c1343773-34cc-4f0a-99ac-06d8e49403b1","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.334Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"be80851f-36ae-4cf1-b0c9-49904429732d","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.336Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"da3240e9-be45-4de9-8f15-39cfeca12a2d","name":"fail-k8s-job-vrx7k-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:03:07.338Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"a2a6c8dc-9d12-41be-b2d8-2e8858cf9429","rayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.347Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"430b61b9-08c1-43df-b614-36ff95780a4d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:07.347Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"430b61b9-08c1-43df-b614-36ff95780a4d","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.348Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"430b61b9-08c1-43df-b614-36ff95780a4d","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:07.348Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"430b61b9-08c1-43df-b614-36ff95780a4d"}
{"level":"info","ts":"2026-07-13T16:03:07.348Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"430b61b9-08c1-43df-b614-36ff95780a4d","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:07.358Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"da3240e9-be45-4de9-8f15-39cfeca12a2d","name":"kuberay-gateway-access","namespace":"test-ns-x5qvg","operation":"created"}
{"level":"info","ts":"2026-07-13T16:03:07.359Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"a2a6c8dc-9d12-41be-b2d8-2e8858cf9429","rayCluster":"fail-k8s-job-vrx7k","podIPs":[]}
{"level":"info","ts":"2026-07-13T16:03:07.368Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"da3240e9-be45-4de9-8f15-39cfeca12a2d","name":"test-ns-x5qvg-fail-k8s-job-vrx7k","operation":"created"}
{"level":"info","ts":"2026-07-13T16:03:07.369Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"a2a6c8dc-9d12-41be-b2d8-2e8858cf9429","rayCluster":"fail-k8s-job-vrx7k","podIPs":[]}
{"level":"error","ts":"2026-07-13T16:03:07.369Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"a2a6c8dc-9d12-41be-b2d8-2e8858cf9429","error":"Certificate.cert-manager.io \"ray-head-cert-fail-k8s-job-vrx7k\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:07.369Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"a2a6c8dc-9d12-41be-b2d8-2e8858cf9429"}
{"level":"error","ts":"2026-07-13T16:03:07.369Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"a2a6c8dc-9d12-41be-b2d8-2e8858cf9429","error":"Certificate.cert-manager.io \"ray-head-cert-fail-k8s-job-vrx7k\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:07.369Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"c418b0eb-9662-4ffc-ab53-2b309e24770d","rayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.421Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"da3240e9-be45-4de9-8f15-39cfeca12a2d","name":"kube-rbac-proxy-config-fail-k8s-job-vrx7k","operation":"created"}
{"level":"error","ts":"2026-07-13T16:03:07.421Z","msg":"Failed to create Ray head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"c418b0eb-9662-4ffc-ab53-2b309e24770d","error":"certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-vrx7k\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayHeadCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:650\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:193\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:03:07.421Z","msg":"Failed to create Ray head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"c418b0eb-9662-4ffc-ab53-2b309e24770d","error":"certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-vrx7k\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:194\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:07.421Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"c418b0eb-9662-4ffc-ab53-2b309e24770d"}
{"level":"error","ts":"2026-07-13T16:03:07.421Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"c418b0eb-9662-4ffc-ab53-2b309e24770d","error":"certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-vrx7k\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:07.421Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"be80851f-36ae-4cf1-b0c9-49904429732d","name":"fail-k8s-job-vrx7k-head"}
{"level":"info","ts":"2026-07-13T16:03:07.421Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d04c6d29-996b-42aa-9f1b-59c963c8920c"}
{"level":"info","ts":"2026-07-13T16:03:07.421Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"84e85182-be09-4ea2-a703-782d2899c0b2","rayCluster":"fail-k8s-job-vrx7k"}
{"level":"error","ts":"2026-07-13T16:03:07.421Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d04c6d29-996b-42aa-9f1b-59c963c8920c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:07.421Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"acf9d45f-9054-4440-9578-b970c3e59921"}
{"level":"info","ts":"2026-07-13T16:03:07.421Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"acf9d45f-9054-4440-9578-b970c3e59921"}
{"level":"info","ts":"2026-07-13T16:03:07.421Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"acf9d45f-9054-4440-9578-b970c3e59921","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:07.421Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"acf9d45f-9054-4440-9578-b970c3e59921","error":"head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found"}
{"level":"error","ts":"2026-07-13T16:03:07.421Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"acf9d45f-9054-4440-9578-b970c3e59921","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:07.421Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"acf9d45f-9054-4440-9578-b970c3e59921","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found"}
{"level":"info","ts":"2026-07-13T16:03:07.421Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"acf9d45f-9054-4440-9578-b970c3e59921","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:07.421Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"acf9d45f-9054-4440-9578-b970c3e59921","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:07.421Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"acf9d45f-9054-4440-9578-b970c3e59921","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:07.422Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"acf9d45f-9054-4440-9578-b970c3e59921"}
{"level":"error","ts":"2026-07-13T16:03:07.422Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"acf9d45f-9054-4440-9578-b970c3e59921","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:03:07.429Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"da3240e9-be45-4de9-8f15-39cfeca12a2d","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-vrx7k\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:07.429Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"da3240e9-be45-4de9-8f15-39cfeca12a2d"}
{"level":"error","ts":"2026-07-13T16:03:07.429Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"da3240e9-be45-4de9-8f15-39cfeca12a2d","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-vrx7k\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:07.429Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"c7402a20-e159-43d1-a1a3-cdc936e06f06","namespacedName":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:07.429Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"c7402a20-e159-43d1-a1a3-cdc936e06f06","mode":"IntegratedOAuth","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.429Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"c7402a20-e159-43d1-a1a3-cdc936e06f06","cluster":"fail-k8s-job-vrx7k"}
{"level":"error","ts":"2026-07-13T16:03:07.430Z","msg":"Failed to create Ray head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"84e85182-be09-4ea2-a703-782d2899c0b2","error":"certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-vrx7k\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayHeadCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:650\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:193\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:03:07.430Z","msg":"Failed to create Ray head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"84e85182-be09-4ea2-a703-782d2899c0b2","error":"certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-vrx7k\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:194\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:07.430Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"84e85182-be09-4ea2-a703-782d2899c0b2"}
{"level":"error","ts":"2026-07-13T16:03:07.430Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"84e85182-be09-4ea2-a703-782d2899c0b2","error":"certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-vrx7k\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:07.431Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"db79282c-d235-439d-815b-05f401bcdcff","rayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.431Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"db79282c-d235-439d-815b-05f401bcdcff"}
{"level":"info","ts":"2026-07-13T16:03:07.431Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d44de4cd-f26f-4787-8d31-8b90bf2ae3fd"}
{"level":"info","ts":"2026-07-13T16:03:07.431Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d44de4cd-f26f-4787-8d31-8b90bf2ae3fd"}
{"level":"info","ts":"2026-07-13T16:03:07.431Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d44de4cd-f26f-4787-8d31-8b90bf2ae3fd","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:07.431Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d44de4cd-f26f-4787-8d31-8b90bf2ae3fd","error":"head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found"}
{"level":"error","ts":"2026-07-13T16:03:07.431Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d44de4cd-f26f-4787-8d31-8b90bf2ae3fd","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:07.432Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d44de4cd-f26f-4787-8d31-8b90bf2ae3fd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found"}
{"level":"info","ts":"2026-07-13T16:03:07.432Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d44de4cd-f26f-4787-8d31-8b90bf2ae3fd","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:07.432Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d44de4cd-f26f-4787-8d31-8b90bf2ae3fd","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:07.432Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d44de4cd-f26f-4787-8d31-8b90bf2ae3fd","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:07.432Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d44de4cd-f26f-4787-8d31-8b90bf2ae3fd"}
{"level":"error","ts":"2026-07-13T16:03:07.432Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d44de4cd-f26f-4787-8d31-8b90bf2ae3fd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:07.433Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"be80851f-36ae-4cf1-b0c9-49904429732d","name":"fail-k8s-job-vrx7k-workers"}
{"level":"info","ts":"2026-07-13T16:03:07.433Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"be80851f-36ae-4cf1-b0c9-49904429732d","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.433Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"c0a33b62-4c42-47cc-8afe-a8dc2f7b7d8a","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.439Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"c7402a20-e159-43d1-a1a3-cdc936e06f06","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.439Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"d92ab998-2205-4806-a157-525abb8e239b","namespacedName":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:07.439Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"d92ab998-2205-4806-a157-525abb8e239b","mode":"IntegratedOAuth","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.521Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"d92ab998-2205-4806-a157-525abb8e239b","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.439Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"4e8f94d8-b4b1-456d-93a0-cfe838ae5e59","rayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.521Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"11d59cb5-b2e7-490a-adae-9c3886236c22"}
{"level":"info","ts":"2026-07-13T16:03:07.521Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"11d59cb5-b2e7-490a-adae-9c3886236c22"}
{"level":"info","ts":"2026-07-13T16:03:07.521Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"11d59cb5-b2e7-490a-adae-9c3886236c22","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:07.521Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"11d59cb5-b2e7-490a-adae-9c3886236c22","error":"head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found"}
{"level":"info","ts":"2026-07-13T16:03:07.521Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"4e8f94d8-b4b1-456d-93a0-cfe838ae5e59"}
{"level":"error","ts":"2026-07-13T16:03:07.521Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"11d59cb5-b2e7-490a-adae-9c3886236c22","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:07.521Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"11d59cb5-b2e7-490a-adae-9c3886236c22","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found"}
{"level":"info","ts":"2026-07-13T16:03:07.521Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"11d59cb5-b2e7-490a-adae-9c3886236c22","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:07.439Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"19909335-2e81-4188-bf1f-decf86ea8baf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:07.521Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"19909335-2e81-4188-bf1f-decf86ea8baf","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.521Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"11d59cb5-b2e7-490a-adae-9c3886236c22","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:07.521Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"19909335-2e81-4188-bf1f-decf86ea8baf","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:07.521Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"11d59cb5-b2e7-490a-adae-9c3886236c22","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:07.521Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"11d59cb5-b2e7-490a-adae-9c3886236c22"}
{"level":"error","ts":"2026-07-13T16:03:07.521Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"11d59cb5-b2e7-490a-adae-9c3886236c22","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:07.522Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"19909335-2e81-4188-bf1f-decf86ea8baf"}
{"level":"info","ts":"2026-07-13T16:03:07.522Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"19909335-2e81-4188-bf1f-decf86ea8baf","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:07.527Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"c0a33b62-4c42-47cc-8afe-a8dc2f7b7d8a","name":"fail-k8s-job-vrx7k-head"}
{"level":"info","ts":"2026-07-13T16:03:07.532Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"d92ab998-2205-4806-a157-525abb8e239b","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.540Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"c0a33b62-4c42-47cc-8afe-a8dc2f7b7d8a","name":"fail-k8s-job-vrx7k-workers"}
{"level":"info","ts":"2026-07-13T16:03:07.540Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"c0a33b62-4c42-47cc-8afe-a8dc2f7b7d8a","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.540Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"4d21b917-f584-4796-b651-6ab9ce51a5f3","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.555Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"4d21b917-f584-4796-b651-6ab9ce51a5f3","name":"fail-k8s-job-vrx7k-head"}
{"level":"info","ts":"2026-07-13T16:03:07.566Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"4d21b917-f584-4796-b651-6ab9ce51a5f3","name":"fail-k8s-job-vrx7k-workers"}
{"level":"info","ts":"2026-07-13T16:03:07.566Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"4d21b917-f584-4796-b651-6ab9ce51a5f3","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:07.621Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"011aab09-4fc6-4fe1-a115-ccf707528f59"}
{"level":"info","ts":"2026-07-13T16:03:07.621Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"011aab09-4fc6-4fe1-a115-ccf707528f59"}
{"level":"info","ts":"2026-07-13T16:03:07.621Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"011aab09-4fc6-4fe1-a115-ccf707528f59","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:07.621Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"011aab09-4fc6-4fe1-a115-ccf707528f59","error":"head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found"}
{"level":"error","ts":"2026-07-13T16:03:07.621Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"011aab09-4fc6-4fe1-a115-ccf707528f59","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:07.621Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"011aab09-4fc6-4fe1-a115-ccf707528f59","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found"}
{"level":"info","ts":"2026-07-13T16:03:07.621Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"011aab09-4fc6-4fe1-a115-ccf707528f59","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:07.621Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"011aab09-4fc6-4fe1-a115-ccf707528f59","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:07.621Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"011aab09-4fc6-4fe1-a115-ccf707528f59","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:07.621Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"011aab09-4fc6-4fe1-a115-ccf707528f59"}
{"level":"error","ts":"2026-07-13T16:03:07.621Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"011aab09-4fc6-4fe1-a115-ccf707528f59","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:07.782Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"81a6106f-3630-4fdc-b689-b55920acb8ba"}
{"level":"info","ts":"2026-07-13T16:03:07.782Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"81a6106f-3630-4fdc-b689-b55920acb8ba"}
{"level":"info","ts":"2026-07-13T16:03:07.782Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"81a6106f-3630-4fdc-b689-b55920acb8ba","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:07.782Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"81a6106f-3630-4fdc-b689-b55920acb8ba","error":"head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found"}
{"level":"error","ts":"2026-07-13T16:03:07.782Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"81a6106f-3630-4fdc-b689-b55920acb8ba","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:07.782Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"81a6106f-3630-4fdc-b689-b55920acb8ba","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found"}
{"level":"info","ts":"2026-07-13T16:03:07.782Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"81a6106f-3630-4fdc-b689-b55920acb8ba","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:07.782Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"81a6106f-3630-4fdc-b689-b55920acb8ba","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:07.782Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"81a6106f-3630-4fdc-b689-b55920acb8ba","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:07.782Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"81a6106f-3630-4fdc-b689-b55920acb8ba"}
{"level":"error","ts":"2026-07-13T16:03:07.782Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"81a6106f-3630-4fdc-b689-b55920acb8ba","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:08.103Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"75a3514c-155c-4a70-aa39-2c1b79e63c26"}
{"level":"info","ts":"2026-07-13T16:03:08.103Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"75a3514c-155c-4a70-aa39-2c1b79e63c26"}
{"level":"info","ts":"2026-07-13T16:03:08.103Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"75a3514c-155c-4a70-aa39-2c1b79e63c26","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:08.103Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"75a3514c-155c-4a70-aa39-2c1b79e63c26","error":"head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found"}
{"level":"error","ts":"2026-07-13T16:03:08.103Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"75a3514c-155c-4a70-aa39-2c1b79e63c26","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:08.103Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"75a3514c-155c-4a70-aa39-2c1b79e63c26","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found"}
{"level":"info","ts":"2026-07-13T16:03:08.103Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"75a3514c-155c-4a70-aa39-2c1b79e63c26","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:08.104Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"75a3514c-155c-4a70-aa39-2c1b79e63c26","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:08.104Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"75a3514c-155c-4a70-aa39-2c1b79e63c26","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:08.104Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"75a3514c-155c-4a70-aa39-2c1b79e63c26"}
{"level":"error","ts":"2026-07-13T16:03:08.104Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"75a3514c-155c-4a70-aa39-2c1b79e63c26","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-vrx7k not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:08.744Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"7e17dee4-54f2-4aa4-94e2-28fb13d16df7"}
{"level":"info","ts":"2026-07-13T16:03:08.744Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"7e17dee4-54f2-4aa4-94e2-28fb13d16df7"}
{"level":"info","ts":"2026-07-13T16:03:08.744Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"7e17dee4-54f2-4aa4-94e2-28fb13d16df7","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:08.744Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"7e17dee4-54f2-4aa4-94e2-28fb13d16df7","error":"worker certificate secret ray-worker-secret-fail-k8s-job-vrx7k not found"}
{"level":"error","ts":"2026-07-13T16:03:08.744Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"7e17dee4-54f2-4aa4-94e2-28fb13d16df7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-fail-k8s-job-vrx7k not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:08.744Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"7e17dee4-54f2-4aa4-94e2-28fb13d16df7","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-fail-k8s-job-vrx7k not found"}
{"level":"info","ts":"2026-07-13T16:03:08.745Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"7e17dee4-54f2-4aa4-94e2-28fb13d16df7","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:08.745Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"7e17dee4-54f2-4aa4-94e2-28fb13d16df7","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:08.745Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"7e17dee4-54f2-4aa4-94e2-28fb13d16df7","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:03:08.745Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"7e17dee4-54f2-4aa4-94e2-28fb13d16df7"}
{"level":"error","ts":"2026-07-13T16:03:08.745Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"7e17dee4-54f2-4aa4-94e2-28fb13d16df7","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-fail-k8s-job-vrx7k not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:10.026Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1"}
{"level":"info","ts":"2026-07-13T16:03:10.026Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1"}
{"level":"info","ts":"2026-07-13T16:03:10.026Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:10.026Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1"}
{"level":"info","ts":"2026-07-13T16:03:10.026Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:03:10.026Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1"}
{"level":"info","ts":"2026-07-13T16:03:10.026Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","mode":"IntegratedOAuth","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:10.026Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","cluster":"fail-k8s-job-vrx7k","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:03:10.026Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","cluster":"fail-k8s-job-vrx7k","authType":"IntegratedOAuth","serviceAccount":"fail-k8s-job-vrx7k-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T16:03:10.026Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-k8s-job-vrx7k","ray.io/group":"headgroup","ray.io/identifier":"fail-k8s-job-vrx7k-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:03:10.026Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-k8s-job-vrx7k-head-svc.test-ns-x5qvg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T16:03:10.026Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-k8s-job-vrx7k-head-svc.test-ns-x5qvg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T16:03:10.026Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-k8s-job-vrx7k-head-svc.test-ns-x5qvg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T16:03:10.026Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","enabled":true}
{"level":"info","ts":"2026-07-13T16:03:10.048Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","name":"fail-k8s-job-vrx7k-head-5fdmc"}
{"level":"info","ts":"2026-07-13T16:03:10.048Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:03:10.048Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:10.048Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:03:10.048Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:10.048Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:03:10.048Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1"}
{"level":"info","ts":"2026-07-13T16:03:10.048Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","nodeType":"worker","rayStartParams":{"address":"fail-k8s-job-vrx7k-head-svc.test-ns-x5qvg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:03:10.048Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","rayStartCmd":"ray start  --address=fail-k8s-job-vrx7k-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:03:10.048Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-k8s-job-vrx7k-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:03:10.048Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","enabled":true}
{"level":"info","ts":"2026-07-13T16:03:10.061Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","name":"fail-k8s-job-vrx7k-small-group-worker-lfkpv"}
{"level":"info","ts":"2026-07-13T16:03:10.061Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:10.061Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-k8s-job-vrx7k-head-svc"},"newHead":{"podName":"fail-k8s-job-vrx7k-head-5fdmc","serviceName":"fail-k8s-job-vrx7k-head-svc"}}
{"level":"info","ts":"2026-07-13T16:03:10.061Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1ee5703a-ef62-4953-a692-4657c04c69d1","name":"fail-k8s-job-vrx7k","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:03:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-vrx7k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-vrx7k-head-5fdmc","serviceName":"fail-k8s-job-vrx7k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:03:10.077Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8e60240b-9754-4182-8c9e-8495d9fa2780"}
{"level":"info","ts":"2026-07-13T16:03:10.077Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8e60240b-9754-4182-8c9e-8495d9fa2780"}
{"level":"info","ts":"2026-07-13T16:03:10.077Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8e60240b-9754-4182-8c9e-8495d9fa2780","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:10.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8e60240b-9754-4182-8c9e-8495d9fa2780","Found 1 head Pod":"fail-k8s-job-vrx7k-head-5fdmc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:03:10.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8e60240b-9754-4182-8c9e-8495d9fa2780","head Pod":"fail-k8s-job-vrx7k-head-5fdmc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-vrx7k-head-5fdmc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:10.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8e60240b-9754-4182-8c9e-8495d9fa2780","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:03:10.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8e60240b-9754-4182-8c9e-8495d9fa2780","worker Pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-vrx7k-small-group-worker-lfkpv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:10.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8e60240b-9754-4182-8c9e-8495d9fa2780","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:10.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8e60240b-9754-4182-8c9e-8495d9fa2780","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:03:10.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8e60240b-9754-4182-8c9e-8495d9fa2780","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:10.078Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"1f9ab3bb-63ca-4ed5-8101-a95f7a19ba11","rayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:10.078Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"ab8bb2e6-278f-4ee0-9b6c-2e965174b67c","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:10.078Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8e60240b-9754-4182-8c9e-8495d9fa2780","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:10.078Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8e60240b-9754-4182-8c9e-8495d9fa2780","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:10.078Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"1f9ab3bb-63ca-4ed5-8101-a95f7a19ba11","rayCluster":"fail-k8s-job-vrx7k","podIPs":[]}
{"level":"info","ts":"2026-07-13T16:03:10.078Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ccfb603e-90f9-447c-aab5-6f3dc8958c13","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:10.121Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ccfb603e-90f9-447c-aab5-6f3dc8958c13","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:10.121Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8e60240b-9754-4182-8c9e-8495d9fa2780","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-k8s-job-vrx7k-head-svc"},"newHead":{"podName":"fail-k8s-job-vrx7k-head-5fdmc","serviceName":"fail-k8s-job-vrx7k-head-svc"}}
{"level":"info","ts":"2026-07-13T16:03:10.121Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ccfb603e-90f9-447c-aab5-6f3dc8958c13","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:10.121Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8e60240b-9754-4182-8c9e-8495d9fa2780","name":"fail-k8s-job-vrx7k","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:03:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-vrx7k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-vrx7k-head-5fdmc","serviceName":"fail-k8s-job-vrx7k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:03:10.121Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ccfb603e-90f9-447c-aab5-6f3dc8958c13"}
{"level":"info","ts":"2026-07-13T16:03:10.121Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ccfb603e-90f9-447c-aab5-6f3dc8958c13","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:10.127Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8e60240b-9754-4182-8c9e-8495d9fa2780","name":"fail-k8s-job-vrx7k","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-vrx7k\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k"}}
{"level":"info","ts":"2026-07-13T16:03:10.127Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8e60240b-9754-4182-8c9e-8495d9fa2780"}
{"level":"error","ts":"2026-07-13T16:03:10.127Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8e60240b-9754-4182-8c9e-8495d9fa2780","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-vrx7k\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:10.127Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"131ce24e-9be5-49ec-9f3b-cd1755c0624a"}
{"level":"info","ts":"2026-07-13T16:03:10.127Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"131ce24e-9be5-49ec-9f3b-cd1755c0624a"}
{"level":"info","ts":"2026-07-13T16:03:10.127Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"131ce24e-9be5-49ec-9f3b-cd1755c0624a","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:10.127Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"131ce24e-9be5-49ec-9f3b-cd1755c0624a","Found 1 head Pod":"fail-k8s-job-vrx7k-head-5fdmc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:03:10.127Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"131ce24e-9be5-49ec-9f3b-cd1755c0624a","head Pod":"fail-k8s-job-vrx7k-head-5fdmc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-vrx7k-head-5fdmc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:10.127Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"131ce24e-9be5-49ec-9f3b-cd1755c0624a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:03:10.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"131ce24e-9be5-49ec-9f3b-cd1755c0624a","worker Pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-vrx7k-small-group-worker-lfkpv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:10.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"131ce24e-9be5-49ec-9f3b-cd1755c0624a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:10.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"131ce24e-9be5-49ec-9f3b-cd1755c0624a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:03:10.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"131ce24e-9be5-49ec-9f3b-cd1755c0624a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:10.128Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"131ce24e-9be5-49ec-9f3b-cd1755c0624a","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:10.128Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"131ce24e-9be5-49ec-9f3b-cd1755c0624a","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:10.128Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"131ce24e-9be5-49ec-9f3b-cd1755c0624a","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T16:03:10.128Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"131ce24e-9be5-49ec-9f3b-cd1755c0624a","name":"fail-k8s-job-vrx7k","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-vrx7k-head-5fdmc","serviceName":"fail-k8s-job-vrx7k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-vrx7k-head-5fdmc","serviceName":"fail-k8s-job-vrx7k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:03:10.130Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"ab8bb2e6-278f-4ee0-9b6c-2e965174b67c","name":"fail-k8s-job-vrx7k-head"}
{"level":"info","ts":"2026-07-13T16:03:10.139Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8b354a82-244b-4eca-b8fc-93181dd5294e"}
{"level":"info","ts":"2026-07-13T16:03:10.139Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8b354a82-244b-4eca-b8fc-93181dd5294e"}
{"level":"info","ts":"2026-07-13T16:03:10.139Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8b354a82-244b-4eca-b8fc-93181dd5294e","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:10.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8b354a82-244b-4eca-b8fc-93181dd5294e","Found 1 head Pod":"fail-k8s-job-vrx7k-head-5fdmc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:03:10.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8b354a82-244b-4eca-b8fc-93181dd5294e","head Pod":"fail-k8s-job-vrx7k-head-5fdmc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-vrx7k-head-5fdmc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:10.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8b354a82-244b-4eca-b8fc-93181dd5294e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:03:10.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8b354a82-244b-4eca-b8fc-93181dd5294e","worker Pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-vrx7k-small-group-worker-lfkpv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:10.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8b354a82-244b-4eca-b8fc-93181dd5294e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:10.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8b354a82-244b-4eca-b8fc-93181dd5294e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:03:10.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8b354a82-244b-4eca-b8fc-93181dd5294e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:10.140Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8b354a82-244b-4eca-b8fc-93181dd5294e","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:10.140Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8b354a82-244b-4eca-b8fc-93181dd5294e","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:10.140Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8b354a82-244b-4eca-b8fc-93181dd5294e","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T16:03:10.140Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8b354a82-244b-4eca-b8fc-93181dd5294e","name":"fail-k8s-job-vrx7k","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-vrx7k-head-5fdmc","serviceName":"fail-k8s-job-vrx7k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-vrx7k-head-5fdmc","serviceName":"fail-k8s-job-vrx7k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:03:10.140Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"89f21459-690e-405b-8904-5ba61c24f63c","rayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:10.140Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"723810d6-11af-4db2-bd79-138a6edc05ca","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:10.140Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"723810d6-11af-4db2-bd79-138a6edc05ca","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:10.140Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"723810d6-11af-4db2-bd79-138a6edc05ca","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:10.140Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"89f21459-690e-405b-8904-5ba61c24f63c","rayCluster":"fail-k8s-job-vrx7k","podIPs":[]}
{"level":"info","ts":"2026-07-13T16:03:10.140Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"723810d6-11af-4db2-bd79-138a6edc05ca"}
{"level":"info","ts":"2026-07-13T16:03:10.140Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"723810d6-11af-4db2-bd79-138a6edc05ca","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:10.142Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"ab8bb2e6-278f-4ee0-9b6c-2e965174b67c","name":"fail-k8s-job-vrx7k-workers"}
{"level":"info","ts":"2026-07-13T16:03:10.142Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"ab8bb2e6-278f-4ee0-9b6c-2e965174b67c","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:10.142Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"92bb0d45-961e-4109-a757-adc6ae94c9f4","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:10.145Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8b354a82-244b-4eca-b8fc-93181dd5294e","name":"fail-k8s-job-vrx7k","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-vrx7k\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k"}}
{"level":"info","ts":"2026-07-13T16:03:10.145Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8b354a82-244b-4eca-b8fc-93181dd5294e"}
{"level":"error","ts":"2026-07-13T16:03:10.145Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"8b354a82-244b-4eca-b8fc-93181dd5294e","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-vrx7k\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:03:10.151Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f4c03ff5-16dd-43fe-a169-e687efc8e45b"}
{"level":"info","ts":"2026-07-13T16:03:10.151Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f4c03ff5-16dd-43fe-a169-e687efc8e45b"}
{"level":"info","ts":"2026-07-13T16:03:10.151Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f4c03ff5-16dd-43fe-a169-e687efc8e45b","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:10.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f4c03ff5-16dd-43fe-a169-e687efc8e45b","Found 1 head Pod":"fail-k8s-job-vrx7k-head-5fdmc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:03:10.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f4c03ff5-16dd-43fe-a169-e687efc8e45b","head Pod":"fail-k8s-job-vrx7k-head-5fdmc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-vrx7k-head-5fdmc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:10.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f4c03ff5-16dd-43fe-a169-e687efc8e45b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:03:10.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f4c03ff5-16dd-43fe-a169-e687efc8e45b","worker Pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-vrx7k-small-group-worker-lfkpv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:10.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f4c03ff5-16dd-43fe-a169-e687efc8e45b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:10.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f4c03ff5-16dd-43fe-a169-e687efc8e45b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:03:10.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f4c03ff5-16dd-43fe-a169-e687efc8e45b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:10.151Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f4c03ff5-16dd-43fe-a169-e687efc8e45b","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:10.151Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f4c03ff5-16dd-43fe-a169-e687efc8e45b","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:10.151Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f4c03ff5-16dd-43fe-a169-e687efc8e45b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:03:10.151Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f4c03ff5-16dd-43fe-a169-e687efc8e45b","seconds":300}
{"level":"info","ts":"2026-07-13T16:03:10.154Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"92bb0d45-961e-4109-a757-adc6ae94c9f4","name":"fail-k8s-job-vrx7k-head"}
{"level":"info","ts":"2026-07-13T16:03:10.162Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"92bb0d45-961e-4109-a757-adc6ae94c9f4","name":"fail-k8s-job-vrx7k-workers"}
{"level":"info","ts":"2026-07-13T16:03:10.163Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"92bb0d45-961e-4109-a757-adc6ae94c9f4","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:10.283Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0208ba8f-7993-44dd-8f3c-4c1c56f56953","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:10.283Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0208ba8f-7993-44dd-8f3c-4c1c56f56953","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:10.283Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0208ba8f-7993-44dd-8f3c-4c1c56f56953","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:10.283Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0208ba8f-7993-44dd-8f3c-4c1c56f56953"}
{"level":"info","ts":"2026-07-13T16:03:10.283Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0208ba8f-7993-44dd-8f3c-4c1c56f56953","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:10.494Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d1f22bda-ec98-480d-acfb-48980dfed5f0"}
{"level":"info","ts":"2026-07-13T16:03:10.494Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d1f22bda-ec98-480d-acfb-48980dfed5f0"}
{"level":"info","ts":"2026-07-13T16:03:10.494Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d1f22bda-ec98-480d-acfb-48980dfed5f0","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:10.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d1f22bda-ec98-480d-acfb-48980dfed5f0","Found 1 head Pod":"fail-k8s-job-vrx7k-head-5fdmc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:03:10.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d1f22bda-ec98-480d-acfb-48980dfed5f0","head Pod":"fail-k8s-job-vrx7k-head-5fdmc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-vrx7k-head-5fdmc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:10.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d1f22bda-ec98-480d-acfb-48980dfed5f0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:03:10.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d1f22bda-ec98-480d-acfb-48980dfed5f0","worker Pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-vrx7k-small-group-worker-lfkpv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:10.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d1f22bda-ec98-480d-acfb-48980dfed5f0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:10.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d1f22bda-ec98-480d-acfb-48980dfed5f0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:03:10.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d1f22bda-ec98-480d-acfb-48980dfed5f0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:10.494Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d1f22bda-ec98-480d-acfb-48980dfed5f0","podName":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:10.494Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d1f22bda-ec98-480d-acfb-48980dfed5f0","podName":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:10.494Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d1f22bda-ec98-480d-acfb-48980dfed5f0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:03:10.495Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d1f22bda-ec98-480d-acfb-48980dfed5f0","seconds":300}
{"level":"info","ts":"2026-07-13T16:03:10.519Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"a26402af-e69b-4a4f-8b41-ad569c43e020"}
{"level":"info","ts":"2026-07-13T16:03:10.519Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"a26402af-e69b-4a4f-8b41-ad569c43e020"}
{"level":"info","ts":"2026-07-13T16:03:10.519Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"a26402af-e69b-4a4f-8b41-ad569c43e020","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:10.519Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"a26402af-e69b-4a4f-8b41-ad569c43e020","Found 1 head Pod":"fail-k8s-job-vrx7k-head-5fdmc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:03:10.519Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"a26402af-e69b-4a4f-8b41-ad569c43e020","head Pod":"fail-k8s-job-vrx7k-head-5fdmc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-vrx7k-head-5fdmc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:10.519Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"a26402af-e69b-4a4f-8b41-ad569c43e020","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:03:10.519Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"a26402af-e69b-4a4f-8b41-ad569c43e020","worker Pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-vrx7k-small-group-worker-lfkpv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:10.519Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"a26402af-e69b-4a4f-8b41-ad569c43e020","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:10.519Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"a26402af-e69b-4a4f-8b41-ad569c43e020","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:03:10.519Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"a26402af-e69b-4a4f-8b41-ad569c43e020","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:10.519Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"a26402af-e69b-4a4f-8b41-ad569c43e020","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:10.519Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"a26402af-e69b-4a4f-8b41-ad569c43e020","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:10.519Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"a26402af-e69b-4a4f-8b41-ad569c43e020","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:03:10.519Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"a26402af-e69b-4a4f-8b41-ad569c43e020","seconds":300}
{"level":"info","ts":"2026-07-13T16:03:10.930Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2602e62f-0399-42c4-aa2d-d54cb239ad90"}
{"level":"info","ts":"2026-07-13T16:03:10.930Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2602e62f-0399-42c4-aa2d-d54cb239ad90"}
{"level":"info","ts":"2026-07-13T16:03:10.930Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2602e62f-0399-42c4-aa2d-d54cb239ad90","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:10.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2602e62f-0399-42c4-aa2d-d54cb239ad90","Found 1 head Pod":"fail-k8s-job-vrx7k-head-5fdmc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:03:10.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2602e62f-0399-42c4-aa2d-d54cb239ad90","head Pod":"fail-k8s-job-vrx7k-head-5fdmc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-vrx7k-head-5fdmc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:10.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2602e62f-0399-42c4-aa2d-d54cb239ad90","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:03:10.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2602e62f-0399-42c4-aa2d-d54cb239ad90","worker Pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-vrx7k-small-group-worker-lfkpv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:10.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2602e62f-0399-42c4-aa2d-d54cb239ad90","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:10.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2602e62f-0399-42c4-aa2d-d54cb239ad90","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:03:10.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2602e62f-0399-42c4-aa2d-d54cb239ad90","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:10.930Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2602e62f-0399-42c4-aa2d-d54cb239ad90","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:10.930Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2602e62f-0399-42c4-aa2d-d54cb239ad90","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:10.930Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2602e62f-0399-42c4-aa2d-d54cb239ad90","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"fail-k8s-job-vrx7k-head-5fdmc","serviceName":"fail-k8s-job-vrx7k-head-svc"},"newHead":{"podIP":"10.133.0.32","serviceIP":"10.133.0.32","podName":"fail-k8s-job-vrx7k-head-5fdmc","serviceName":"fail-k8s-job-vrx7k-head-svc"}}
{"level":"info","ts":"2026-07-13T16:03:10.930Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2602e62f-0399-42c4-aa2d-d54cb239ad90","name":"fail-k8s-job-vrx7k","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-vrx7k-head-5fdmc","serviceName":"fail-k8s-job-vrx7k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.32","serviceIP":"10.133.0.32","podName":"fail-k8s-job-vrx7k-head-5fdmc","serviceName":"fail-k8s-job-vrx7k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:03:10.941Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"dabff747-fc6c-43bb-97f5-a3396bccd18a","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:10.941Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"1999760f-d02a-403a-9be8-73a6676f39dd","rayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:10.942Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"1999760f-d02a-403a-9be8-73a6676f39dd","pod":"fail-k8s-job-vrx7k-head-5fdmc","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-13T16:03:10.942Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b928dd6a-4cc7-4c34-afd4-27837677d197","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:10.942Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b928dd6a-4cc7-4c34-afd4-27837677d197","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:10.942Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b928dd6a-4cc7-4c34-afd4-27837677d197","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:10.942Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b928dd6a-4cc7-4c34-afd4-27837677d197"}
{"level":"info","ts":"2026-07-13T16:03:10.942Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b928dd6a-4cc7-4c34-afd4-27837677d197","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:10.952Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"dabff747-fc6c-43bb-97f5-a3396bccd18a","name":"fail-k8s-job-vrx7k-head"}
{"level":"info","ts":"2026-07-13T16:03:10.953Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"1999760f-d02a-403a-9be8-73a6676f39dd","certificate":"ray-head-cert-fail-k8s-job-vrx7k","dnsNames":["fail-k8s-job-vrx7k-head-svc","localhost","fail-k8s-job-vrx7k-head-svc.test-ns-x5qvg.svc","fail-k8s-job-vrx7k-head-svc.test-ns-x5qvg.svc.cluster.local"],"ipAddresses":["10.133.0.32","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:03:10.963Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"dabff747-fc6c-43bb-97f5-a3396bccd18a","name":"fail-k8s-job-vrx7k-workers"}
{"level":"info","ts":"2026-07-13T16:03:10.963Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"dabff747-fc6c-43bb-97f5-a3396bccd18a","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:10.964Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"1999760f-d02a-403a-9be8-73a6676f39dd","certificate":"ray-worker-cert-fail-k8s-job-vrx7k","dnsNames":["fail-k8s-job-vrx7k-worker-svc","localhost","fail-k8s-job-vrx7k-small-group","fail-k8s-job-vrx7k-worker-svc.test-ns-x5qvg.svc","fail-k8s-job-vrx7k-worker-svc.test-ns-x5qvg.svc.cluster.local","*.fail-k8s-job-vrx7k-worker-svc.test-ns-x5qvg.svc","*.fail-k8s-job-vrx7k-worker-svc.test-ns-x5qvg.svc.cluster.local","*-worker-*.test-ns-x5qvg.svc","*-worker-*.test-ns-x5qvg.svc.cluster.local"],"ipAddresses":["10.133.0.32","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:03:10.964Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"1999760f-d02a-403a-9be8-73a6676f39dd","rayCluster":"fail-k8s-job-vrx7k","podIPs":["10.133.0.32"]}
{"level":"info","ts":"2026-07-13T16:03:11.244Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b06e5095-87df-4f38-8011-163eb3ca7f51"}
{"level":"info","ts":"2026-07-13T16:03:11.244Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b06e5095-87df-4f38-8011-163eb3ca7f51"}
{"level":"info","ts":"2026-07-13T16:03:11.244Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b06e5095-87df-4f38-8011-163eb3ca7f51","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:11.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b06e5095-87df-4f38-8011-163eb3ca7f51","Found 1 head Pod":"fail-k8s-job-vrx7k-head-5fdmc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:03:11.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b06e5095-87df-4f38-8011-163eb3ca7f51","head Pod":"fail-k8s-job-vrx7k-head-5fdmc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-vrx7k-head-5fdmc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:11.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b06e5095-87df-4f38-8011-163eb3ca7f51","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:03:11.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b06e5095-87df-4f38-8011-163eb3ca7f51","worker Pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-vrx7k-small-group-worker-lfkpv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:11.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b06e5095-87df-4f38-8011-163eb3ca7f51","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:11.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b06e5095-87df-4f38-8011-163eb3ca7f51","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:03:11.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b06e5095-87df-4f38-8011-163eb3ca7f51","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:11.245Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b06e5095-87df-4f38-8011-163eb3ca7f51","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:11.245Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b06e5095-87df-4f38-8011-163eb3ca7f51","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:11.245Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b06e5095-87df-4f38-8011-163eb3ca7f51","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:03:11.245Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b06e5095-87df-4f38-8011-163eb3ca7f51","seconds":300}
{"level":"info","ts":"2026-07-13T16:03:12.942Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d242c982-aab9-426e-86a8-469b0f6cd4a5"}
{"level":"info","ts":"2026-07-13T16:03:12.942Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d242c982-aab9-426e-86a8-469b0f6cd4a5"}
{"level":"info","ts":"2026-07-13T16:03:12.942Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d242c982-aab9-426e-86a8-469b0f6cd4a5","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:12.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d242c982-aab9-426e-86a8-469b0f6cd4a5","Found 1 head Pod":"fail-k8s-job-vrx7k-head-5fdmc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:03:12.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d242c982-aab9-426e-86a8-469b0f6cd4a5","head Pod":"fail-k8s-job-vrx7k-head-5fdmc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-vrx7k-head-5fdmc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:12.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d242c982-aab9-426e-86a8-469b0f6cd4a5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:03:12.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d242c982-aab9-426e-86a8-469b0f6cd4a5","worker Pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-vrx7k-small-group-worker-lfkpv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:12.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d242c982-aab9-426e-86a8-469b0f6cd4a5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:12.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d242c982-aab9-426e-86a8-469b0f6cd4a5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:03:12.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d242c982-aab9-426e-86a8-469b0f6cd4a5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:12.943Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d242c982-aab9-426e-86a8-469b0f6cd4a5","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:12.943Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d242c982-aab9-426e-86a8-469b0f6cd4a5","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:12.943Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d242c982-aab9-426e-86a8-469b0f6cd4a5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:03:12.943Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"d242c982-aab9-426e-86a8-469b0f6cd4a5","seconds":300}
{"level":"info","ts":"2026-07-13T16:03:13.284Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"8a1e4ff7-d857-434c-8404-ddd56dc9cfe9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:13.284Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"8a1e4ff7-d857-434c-8404-ddd56dc9cfe9","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:13.284Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"8a1e4ff7-d857-434c-8404-ddd56dc9cfe9","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:13.284Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"8a1e4ff7-d857-434c-8404-ddd56dc9cfe9"}
{"level":"info","ts":"2026-07-13T16:03:13.284Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"8a1e4ff7-d857-434c-8404-ddd56dc9cfe9","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:15.949Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2b417eb4-e12f-4b1a-9ecc-7c476a71dde0"}
{"level":"info","ts":"2026-07-13T16:03:15.949Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2b417eb4-e12f-4b1a-9ecc-7c476a71dde0"}
{"level":"info","ts":"2026-07-13T16:03:15.949Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2b417eb4-e12f-4b1a-9ecc-7c476a71dde0","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:15.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2b417eb4-e12f-4b1a-9ecc-7c476a71dde0","Found 1 head Pod":"fail-k8s-job-vrx7k-head-5fdmc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:03:15.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2b417eb4-e12f-4b1a-9ecc-7c476a71dde0","head Pod":"fail-k8s-job-vrx7k-head-5fdmc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-vrx7k-head-5fdmc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:15.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2b417eb4-e12f-4b1a-9ecc-7c476a71dde0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:03:15.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2b417eb4-e12f-4b1a-9ecc-7c476a71dde0","worker Pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-vrx7k-small-group-worker-lfkpv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:15.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2b417eb4-e12f-4b1a-9ecc-7c476a71dde0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:15.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2b417eb4-e12f-4b1a-9ecc-7c476a71dde0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:03:15.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2b417eb4-e12f-4b1a-9ecc-7c476a71dde0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:15.949Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2b417eb4-e12f-4b1a-9ecc-7c476a71dde0","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:15.949Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2b417eb4-e12f-4b1a-9ecc-7c476a71dde0","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:15.949Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2b417eb4-e12f-4b1a-9ecc-7c476a71dde0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:03:15.949Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2b417eb4-e12f-4b1a-9ecc-7c476a71dde0","seconds":300}
{"level":"info","ts":"2026-07-13T16:03:16.285Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1b61f360-3770-4658-be1b-cff8ce2e47bc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:16.285Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1b61f360-3770-4658-be1b-cff8ce2e47bc","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:16.285Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1b61f360-3770-4658-be1b-cff8ce2e47bc","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:16.285Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1b61f360-3770-4658-be1b-cff8ce2e47bc"}
{"level":"info","ts":"2026-07-13T16:03:16.285Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1b61f360-3770-4658-be1b-cff8ce2e47bc","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:16.970Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b6d1c7bf-3656-4b7f-8750-305987dd3823"}
{"level":"info","ts":"2026-07-13T16:03:16.971Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b6d1c7bf-3656-4b7f-8750-305987dd3823"}
{"level":"info","ts":"2026-07-13T16:03:16.971Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b6d1c7bf-3656-4b7f-8750-305987dd3823","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:16.971Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b6d1c7bf-3656-4b7f-8750-305987dd3823","Found 1 head Pod":"fail-k8s-job-vrx7k-head-5fdmc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:03:16.971Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b6d1c7bf-3656-4b7f-8750-305987dd3823","head Pod":"fail-k8s-job-vrx7k-head-5fdmc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-vrx7k-head-5fdmc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:16.971Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b6d1c7bf-3656-4b7f-8750-305987dd3823","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:03:16.971Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b6d1c7bf-3656-4b7f-8750-305987dd3823","worker Pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-vrx7k-small-group-worker-lfkpv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:16.971Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b6d1c7bf-3656-4b7f-8750-305987dd3823","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:16.971Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b6d1c7bf-3656-4b7f-8750-305987dd3823","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:03:16.971Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b6d1c7bf-3656-4b7f-8750-305987dd3823","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:16.971Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b6d1c7bf-3656-4b7f-8750-305987dd3823","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:03:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:03:16.971Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b6d1c7bf-3656-4b7f-8750-305987dd3823","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:03:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:03:16.971Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b6d1c7bf-3656-4b7f-8750-305987dd3823","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T16:03:16.971Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b6d1c7bf-3656-4b7f-8750-305987dd3823","name":"fail-k8s-job-vrx7k","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.32","serviceIP":"10.133.0.32","podName":"fail-k8s-job-vrx7k-head-5fdmc","serviceName":"fail-k8s-job-vrx7k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:03:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.32","serviceIP":"10.133.0.32","podName":"fail-k8s-job-vrx7k-head-5fdmc","serviceName":"fail-k8s-job-vrx7k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:03:16.984Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"c144c83c-9983-4d0f-83ce-450bd36d179a","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:16.984Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"139d4d6a-4461-420c-a3be-554012f3ac03","rayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:16.984Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"139d4d6a-4461-420c-a3be-554012f3ac03","pod":"fail-k8s-job-vrx7k-head-5fdmc","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-13T16:03:16.984Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"139d4d6a-4461-420c-a3be-554012f3ac03","pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","ip":"10.134.0.33"}
{"level":"info","ts":"2026-07-13T16:03:16.984Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c7e4fc77-c74a-4b2a-b4e3-505a4072c754","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:16.984Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c7e4fc77-c74a-4b2a-b4e3-505a4072c754","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:16.984Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c7e4fc77-c74a-4b2a-b4e3-505a4072c754","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:16.984Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c7e4fc77-c74a-4b2a-b4e3-505a4072c754"}
{"level":"info","ts":"2026-07-13T16:03:16.984Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c7e4fc77-c74a-4b2a-b4e3-505a4072c754","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:16.995Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"c144c83c-9983-4d0f-83ce-450bd36d179a","name":"fail-k8s-job-vrx7k-head"}
{"level":"info","ts":"2026-07-13T16:03:16.996Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"139d4d6a-4461-420c-a3be-554012f3ac03","certificate":"ray-head-cert-fail-k8s-job-vrx7k","dnsNames":["fail-k8s-job-vrx7k-head-svc","localhost","fail-k8s-job-vrx7k-head-svc.test-ns-x5qvg.svc","fail-k8s-job-vrx7k-head-svc.test-ns-x5qvg.svc.cluster.local"],"ipAddresses":["10.133.0.32","10.134.0.33","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:03:17.007Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"c144c83c-9983-4d0f-83ce-450bd36d179a","name":"fail-k8s-job-vrx7k-workers"}
{"level":"info","ts":"2026-07-13T16:03:17.007Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"c144c83c-9983-4d0f-83ce-450bd36d179a","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:17.008Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"139d4d6a-4461-420c-a3be-554012f3ac03","certificate":"ray-worker-cert-fail-k8s-job-vrx7k","dnsNames":["fail-k8s-job-vrx7k-worker-svc","localhost","fail-k8s-job-vrx7k-small-group","fail-k8s-job-vrx7k-worker-svc.test-ns-x5qvg.svc","fail-k8s-job-vrx7k-worker-svc.test-ns-x5qvg.svc.cluster.local","*.fail-k8s-job-vrx7k-worker-svc.test-ns-x5qvg.svc","*.fail-k8s-job-vrx7k-worker-svc.test-ns-x5qvg.svc.cluster.local","*-worker-*.test-ns-x5qvg.svc","*-worker-*.test-ns-x5qvg.svc.cluster.local"],"ipAddresses":["10.133.0.32","10.134.0.33","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:03:17.008Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"139d4d6a-4461-420c-a3be-554012f3ac03","rayCluster":"fail-k8s-job-vrx7k","podIPs":["10.133.0.32","10.134.0.33"]}
{"level":"info","ts":"2026-07-13T16:03:18.984Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2692d245-ab8f-4649-8f13-a336d2a7c6ac"}
{"level":"info","ts":"2026-07-13T16:03:18.984Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2692d245-ab8f-4649-8f13-a336d2a7c6ac"}
{"level":"info","ts":"2026-07-13T16:03:18.984Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2692d245-ab8f-4649-8f13-a336d2a7c6ac","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:18.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2692d245-ab8f-4649-8f13-a336d2a7c6ac","Found 1 head Pod":"fail-k8s-job-vrx7k-head-5fdmc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:03:18.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2692d245-ab8f-4649-8f13-a336d2a7c6ac","head Pod":"fail-k8s-job-vrx7k-head-5fdmc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-vrx7k-head-5fdmc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:18.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2692d245-ab8f-4649-8f13-a336d2a7c6ac","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:03:18.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2692d245-ab8f-4649-8f13-a336d2a7c6ac","worker Pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-vrx7k-small-group-worker-lfkpv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:18.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2692d245-ab8f-4649-8f13-a336d2a7c6ac","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:18.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2692d245-ab8f-4649-8f13-a336d2a7c6ac","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:03:18.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2692d245-ab8f-4649-8f13-a336d2a7c6ac","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:18.984Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2692d245-ab8f-4649-8f13-a336d2a7c6ac","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:03:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:03:18.984Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2692d245-ab8f-4649-8f13-a336d2a7c6ac","podName":"fail-k8s-job-vrx7k-head-5fdmc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:03:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:03:18.984Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2692d245-ab8f-4649-8f13-a336d2a7c6ac","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:03:18.984Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"2692d245-ab8f-4649-8f13-a336d2a7c6ac","seconds":300}
{"level":"info","ts":"2026-07-13T16:03:19.286Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"f9387861-ec0d-4ca1-926b-9bd357ffaf40","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:19.286Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"f9387861-ec0d-4ca1-926b-9bd357ffaf40","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:19.286Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"f9387861-ec0d-4ca1-926b-9bd357ffaf40","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:19.287Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"f9387861-ec0d-4ca1-926b-9bd357ffaf40"}
{"level":"info","ts":"2026-07-13T16:03:19.287Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"f9387861-ec0d-4ca1-926b-9bd357ffaf40","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:22.287Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"82f3685e-7072-4f82-a7e9-b7426be30065","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:22.287Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"82f3685e-7072-4f82-a7e9-b7426be30065","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:22.287Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"82f3685e-7072-4f82-a7e9-b7426be30065","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:22.288Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"82f3685e-7072-4f82-a7e9-b7426be30065"}
{"level":"info","ts":"2026-07-13T16:03:22.288Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"82f3685e-7072-4f82-a7e9-b7426be30065","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:25.288Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"7fd6688b-7c8b-4885-9304-75fca4f7b941","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:25.288Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"7fd6688b-7c8b-4885-9304-75fca4f7b941","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:25.288Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"7fd6688b-7c8b-4885-9304-75fca4f7b941","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:25.288Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"7fd6688b-7c8b-4885-9304-75fca4f7b941"}
{"level":"info","ts":"2026-07-13T16:03:25.288Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"7fd6688b-7c8b-4885-9304-75fca4f7b941","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:27.987Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"3f7a0220-82bf-4d85-bf6c-2eb6b899844e"}
{"level":"info","ts":"2026-07-13T16:03:27.987Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"3f7a0220-82bf-4d85-bf6c-2eb6b899844e"}
{"level":"info","ts":"2026-07-13T16:03:27.987Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"3f7a0220-82bf-4d85-bf6c-2eb6b899844e","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:27.987Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"3f7a0220-82bf-4d85-bf6c-2eb6b899844e","Found 1 head Pod":"fail-k8s-job-vrx7k-head-5fdmc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:03:27.987Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"3f7a0220-82bf-4d85-bf6c-2eb6b899844e","head Pod":"fail-k8s-job-vrx7k-head-5fdmc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-vrx7k-head-5fdmc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:27.987Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"3f7a0220-82bf-4d85-bf6c-2eb6b899844e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:03:27.987Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"3f7a0220-82bf-4d85-bf6c-2eb6b899844e","worker Pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-vrx7k-small-group-worker-lfkpv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:27.987Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"3f7a0220-82bf-4d85-bf6c-2eb6b899844e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:27.987Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"3f7a0220-82bf-4d85-bf6c-2eb6b899844e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:03:27.987Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"3f7a0220-82bf-4d85-bf6c-2eb6b899844e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:27.987Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"3f7a0220-82bf-4d85-bf6c-2eb6b899844e","podName":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:27.987Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"3f7a0220-82bf-4d85-bf6c-2eb6b899844e","podName":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:27.987Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"3f7a0220-82bf-4d85-bf6c-2eb6b899844e","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:03:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T16:03:27.987Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"3f7a0220-82bf-4d85-bf6c-2eb6b899844e","name":"fail-k8s-job-vrx7k","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:03:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.32","serviceIP":"10.133.0.32","podName":"fail-k8s-job-vrx7k-head-5fdmc","serviceName":"fail-k8s-job-vrx7k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:03:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.32","serviceIP":"10.133.0.32","podName":"fail-k8s-job-vrx7k-head-5fdmc","serviceName":"fail-k8s-job-vrx7k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:03:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:03:28.000Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"903c3d36-7e30-4ebf-aa57-6d32551024c7","rayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:28.000Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"10217965-6d9a-4a17-b766-cb22a509a93c","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:28.000Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a44ff075-1338-4fa8-9c6c-b100d9da2dfc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:28.000Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a44ff075-1338-4fa8-9c6c-b100d9da2dfc","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:28.000Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"903c3d36-7e30-4ebf-aa57-6d32551024c7","pod":"fail-k8s-job-vrx7k-head-5fdmc","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-13T16:03:28.000Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"903c3d36-7e30-4ebf-aa57-6d32551024c7","pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","ip":"10.134.0.33"}
{"level":"info","ts":"2026-07-13T16:03:28.000Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"903c3d36-7e30-4ebf-aa57-6d32551024c7","rayCluster":"fail-k8s-job-vrx7k","podIPs":["10.133.0.32","10.134.0.33"]}
{"level":"info","ts":"2026-07-13T16:03:28.000Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a44ff075-1338-4fa8-9c6c-b100d9da2dfc","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:28.000Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a44ff075-1338-4fa8-9c6c-b100d9da2dfc"}
{"level":"info","ts":"2026-07-13T16:03:28.000Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a44ff075-1338-4fa8-9c6c-b100d9da2dfc","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:28.008Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"10217965-6d9a-4a17-b766-cb22a509a93c","name":"fail-k8s-job-vrx7k-head"}
{"level":"info","ts":"2026-07-13T16:03:28.018Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"10217965-6d9a-4a17-b766-cb22a509a93c","name":"fail-k8s-job-vrx7k-workers"}
{"level":"info","ts":"2026-07-13T16:03:28.018Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"10217965-6d9a-4a17-b766-cb22a509a93c","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:28.289Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"85520a41-d365-4658-ad44-0c906ea5e295","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:28.289Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"85520a41-d365-4658-ad44-0c906ea5e295","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:28.289Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"85520a41-d365-4658-ad44-0c906ea5e295","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:28.289Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"85520a41-d365-4658-ad44-0c906ea5e295"}
{"level":"info","ts":"2026-07-13T16:03:28.289Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"85520a41-d365-4658-ad44-0c906ea5e295","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:30.000Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"87e5954b-61d7-4a33-96d4-b54bc9891c9e"}
{"level":"info","ts":"2026-07-13T16:03:30.000Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"87e5954b-61d7-4a33-96d4-b54bc9891c9e"}
{"level":"info","ts":"2026-07-13T16:03:30.000Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"87e5954b-61d7-4a33-96d4-b54bc9891c9e","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:03:30.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"87e5954b-61d7-4a33-96d4-b54bc9891c9e","Found 1 head Pod":"fail-k8s-job-vrx7k-head-5fdmc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:03:30.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"87e5954b-61d7-4a33-96d4-b54bc9891c9e","head Pod":"fail-k8s-job-vrx7k-head-5fdmc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-vrx7k-head-5fdmc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:30.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"87e5954b-61d7-4a33-96d4-b54bc9891c9e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:03:30.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"87e5954b-61d7-4a33-96d4-b54bc9891c9e","worker Pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-vrx7k-small-group-worker-lfkpv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:03:30.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"87e5954b-61d7-4a33-96d4-b54bc9891c9e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:30.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"87e5954b-61d7-4a33-96d4-b54bc9891c9e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:03:30.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"87e5954b-61d7-4a33-96d4-b54bc9891c9e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:03:30.000Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"87e5954b-61d7-4a33-96d4-b54bc9891c9e","podName":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:30.000Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"87e5954b-61d7-4a33-96d4-b54bc9891c9e","podName":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:03:30.000Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"87e5954b-61d7-4a33-96d4-b54bc9891c9e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:03:30.000Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"87e5954b-61d7-4a33-96d4-b54bc9891c9e","seconds":300}
{"level":"info","ts":"2026-07-13T16:03:31.290Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"122eccab-9479-41a5-b185-94afeb316782","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:31.290Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"122eccab-9479-41a5-b185-94afeb316782","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:31.290Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"122eccab-9479-41a5-b185-94afeb316782","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:31.290Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"122eccab-9479-41a5-b185-94afeb316782"}
{"level":"info","ts":"2026-07-13T16:03:31.290Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"122eccab-9479-41a5-b185-94afeb316782","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:34.290Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ffe338d9-8d43-462b-a0b1-5a64b8f092d4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:34.290Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ffe338d9-8d43-462b-a0b1-5a64b8f092d4","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:34.290Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ffe338d9-8d43-462b-a0b1-5a64b8f092d4","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:34.291Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ffe338d9-8d43-462b-a0b1-5a64b8f092d4"}
{"level":"info","ts":"2026-07-13T16:03:34.291Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ffe338d9-8d43-462b-a0b1-5a64b8f092d4","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:34.945Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"d4ae93b9-4508-4b1f-9c57-ec0aa1a84a33","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:03:34.945Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"d4ae93b9-4508-4b1f-9c57-ec0aa1a84a33","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T16:03:34.945Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"d4ae93b9-4508-4b1f-9c57-ec0aa1a84a33","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T16:03:34.945Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"d4ae93b9-4508-4b1f-9c57-ec0aa1a84a33","rayCluster":"raycluster","podIPs":["10.133.0.18","10.134.0.25"]}
{"level":"info","ts":"2026-07-13T16:03:37.292Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d610af73-8b02-4c84-bca1-7a9025e8ea61","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:37.292Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d610af73-8b02-4c84-bca1-7a9025e8ea61","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:37.292Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d610af73-8b02-4c84-bca1-7a9025e8ea61","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:37.292Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d610af73-8b02-4c84-bca1-7a9025e8ea61"}
{"level":"info","ts":"2026-07-13T16:03:37.292Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d610af73-8b02-4c84-bca1-7a9025e8ea61","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:37.432Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"2b353a40-fc9b-4fca-b116-6cfcb7953006","rayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:37.432Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"2b353a40-fc9b-4fca-b116-6cfcb7953006","pod":"fail-k8s-job-vrx7k-head-5fdmc","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-13T16:03:37.432Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"2b353a40-fc9b-4fca-b116-6cfcb7953006","pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","ip":"10.134.0.33"}
{"level":"info","ts":"2026-07-13T16:03:37.432Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"2b353a40-fc9b-4fca-b116-6cfcb7953006","rayCluster":"fail-k8s-job-vrx7k","podIPs":["10.133.0.32","10.134.0.33"]}
{"level":"info","ts":"2026-07-13T16:03:40.293Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"29535e4e-5d8a-4032-8acc-76174c40f4e6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:40.293Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"29535e4e-5d8a-4032-8acc-76174c40f4e6","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:40.293Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"29535e4e-5d8a-4032-8acc-76174c40f4e6","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:40.293Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"29535e4e-5d8a-4032-8acc-76174c40f4e6"}
{"level":"info","ts":"2026-07-13T16:03:40.293Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"29535e4e-5d8a-4032-8acc-76174c40f4e6","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:43.293Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"723bb226-c333-493d-bdd9-16567f25d346","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:43.293Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"723bb226-c333-493d-bdd9-16567f25d346","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:43.294Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"723bb226-c333-493d-bdd9-16567f25d346","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:43.294Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"723bb226-c333-493d-bdd9-16567f25d346"}
{"level":"info","ts":"2026-07-13T16:03:43.294Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"723bb226-c333-493d-bdd9-16567f25d346","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:46.294Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"bc0f5d1b-81ea-4047-b654-9ff21ccde3c1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:46.294Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"bc0f5d1b-81ea-4047-b654-9ff21ccde3c1","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:46.294Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"bc0f5d1b-81ea-4047-b654-9ff21ccde3c1","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:46.295Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"bc0f5d1b-81ea-4047-b654-9ff21ccde3c1"}
{"level":"info","ts":"2026-07-13T16:03:46.295Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"bc0f5d1b-81ea-4047-b654-9ff21ccde3c1","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:49.295Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"324d0aaa-30f2-4aba-81a7-db1e928473db","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:49.295Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"324d0aaa-30f2-4aba-81a7-db1e928473db","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:49.295Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"324d0aaa-30f2-4aba-81a7-db1e928473db","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:49.295Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"324d0aaa-30f2-4aba-81a7-db1e928473db"}
{"level":"info","ts":"2026-07-13T16:03:49.295Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"324d0aaa-30f2-4aba-81a7-db1e928473db","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:52.296Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c7dbcbd6-95f1-4b17-9f1a-6ba0d43b7c0a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:52.296Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c7dbcbd6-95f1-4b17-9f1a-6ba0d43b7c0a","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:52.296Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c7dbcbd6-95f1-4b17-9f1a-6ba0d43b7c0a","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:52.296Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c7dbcbd6-95f1-4b17-9f1a-6ba0d43b7c0a"}
{"level":"info","ts":"2026-07-13T16:03:52.296Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c7dbcbd6-95f1-4b17-9f1a-6ba0d43b7c0a","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:55.297Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"36ae72c2-c90f-43c8-9dc8-c9c341e8db6f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:55.297Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"36ae72c2-c90f-43c8-9dc8-c9c341e8db6f","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:55.297Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"36ae72c2-c90f-43c8-9dc8-c9c341e8db6f","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:55.297Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"36ae72c2-c90f-43c8-9dc8-c9c341e8db6f"}
{"level":"info","ts":"2026-07-13T16:03:55.297Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"36ae72c2-c90f-43c8-9dc8-c9c341e8db6f","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:03:58.297Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0880110b-3ee4-4cb3-aad4-802b37a12a5c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:03:58.297Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0880110b-3ee4-4cb3-aad4-802b37a12a5c","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:03:58.297Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0880110b-3ee4-4cb3-aad4-802b37a12a5c","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:03:58.298Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0880110b-3ee4-4cb3-aad4-802b37a12a5c"}
{"level":"info","ts":"2026-07-13T16:03:58.298Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0880110b-3ee4-4cb3-aad4-802b37a12a5c","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:04:01.298Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b5d07a2d-a16f-424c-85b6-e9a22683bc16","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:01.298Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b5d07a2d-a16f-424c-85b6-e9a22683bc16","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:01.298Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b5d07a2d-a16f-424c-85b6-e9a22683bc16","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:01.298Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b5d07a2d-a16f-424c-85b6-e9a22683bc16"}
{"level":"info","ts":"2026-07-13T16:04:01.298Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b5d07a2d-a16f-424c-85b6-e9a22683bc16","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:04:04.298Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"382802b5-cce4-4677-8564-174b5554b32d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:04.299Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"382802b5-cce4-4677-8564-174b5554b32d","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:04.299Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"382802b5-cce4-4677-8564-174b5554b32d","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:04.299Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"382802b5-cce4-4677-8564-174b5554b32d"}
{"level":"info","ts":"2026-07-13T16:04:04.299Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"382802b5-cce4-4677-8564-174b5554b32d","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:04:07.299Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"37100cb4-096b-4009-ab2a-78a92e9b0edf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:07.299Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"37100cb4-096b-4009-ab2a-78a92e9b0edf","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:07.299Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"37100cb4-096b-4009-ab2a-78a92e9b0edf","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:07.299Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"37100cb4-096b-4009-ab2a-78a92e9b0edf"}
{"level":"info","ts":"2026-07-13T16:04:07.299Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"37100cb4-096b-4009-ab2a-78a92e9b0edf","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:04:10.300Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0b5d71dc-c337-4de6-976f-be00c695a3b6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:10.301Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0b5d71dc-c337-4de6-976f-be00c695a3b6","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:10.301Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0b5d71dc-c337-4de6-976f-be00c695a3b6","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:10.301Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0b5d71dc-c337-4de6-976f-be00c695a3b6"}
{"level":"info","ts":"2026-07-13T16:04:10.301Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0b5d71dc-c337-4de6-976f-be00c695a3b6","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:04:13.302Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d5bd290b-9da7-41fb-b587-b28d44c75546","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:13.302Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d5bd290b-9da7-41fb-b587-b28d44c75546","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:13.302Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d5bd290b-9da7-41fb-b587-b28d44c75546","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:13.302Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d5bd290b-9da7-41fb-b587-b28d44c75546"}
{"level":"info","ts":"2026-07-13T16:04:13.302Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d5bd290b-9da7-41fb-b587-b28d44c75546","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:04:16.302Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d61965fb-4429-443f-83d8-be11e89c82d4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:16.302Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d61965fb-4429-443f-83d8-be11e89c82d4","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:16.302Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d61965fb-4429-443f-83d8-be11e89c82d4","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:16.302Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d61965fb-4429-443f-83d8-be11e89c82d4"}
{"level":"info","ts":"2026-07-13T16:04:16.303Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d61965fb-4429-443f-83d8-be11e89c82d4","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:04:19.303Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ac2a227c-c9c4-449a-be47-b8a020b259c8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:19.303Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ac2a227c-c9c4-449a-be47-b8a020b259c8","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:19.303Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ac2a227c-c9c4-449a-be47-b8a020b259c8","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:19.303Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ac2a227c-c9c4-449a-be47-b8a020b259c8"}
{"level":"info","ts":"2026-07-13T16:04:19.303Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ac2a227c-c9c4-449a-be47-b8a020b259c8","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:04:22.304Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a086f87b-5915-4c93-bc3f-26ef5309b64b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:22.304Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a086f87b-5915-4c93-bc3f-26ef5309b64b","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:22.304Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a086f87b-5915-4c93-bc3f-26ef5309b64b","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:22.304Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a086f87b-5915-4c93-bc3f-26ef5309b64b"}
{"level":"info","ts":"2026-07-13T16:04:22.304Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a086f87b-5915-4c93-bc3f-26ef5309b64b","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:04:25.305Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"146853ee-a77f-484c-b15e-2f35dc070f24","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:25.305Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"146853ee-a77f-484c-b15e-2f35dc070f24","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:25.305Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"146853ee-a77f-484c-b15e-2f35dc070f24","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:25.305Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"146853ee-a77f-484c-b15e-2f35dc070f24"}
{"level":"info","ts":"2026-07-13T16:04:25.305Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"146853ee-a77f-484c-b15e-2f35dc070f24","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:04:26.557Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f70ebed7-0f79-4d61-8260-5df9fcecd06a"}
{"level":"info","ts":"2026-07-13T16:04:26.557Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f70ebed7-0f79-4d61-8260-5df9fcecd06a"}
{"level":"info","ts":"2026-07-13T16:04:26.557Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f70ebed7-0f79-4d61-8260-5df9fcecd06a","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:04:26.557Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f70ebed7-0f79-4d61-8260-5df9fcecd06a","Found 1 head Pod":"fail-k8s-job-vrx7k-head-5fdmc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:04:26.558Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f70ebed7-0f79-4d61-8260-5df9fcecd06a","head Pod":"fail-k8s-job-vrx7k-head-5fdmc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-vrx7k-head-5fdmc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:04:26.558Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f70ebed7-0f79-4d61-8260-5df9fcecd06a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:04:26.558Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f70ebed7-0f79-4d61-8260-5df9fcecd06a","worker Pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-vrx7k-small-group-worker-lfkpv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:04:26.558Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f70ebed7-0f79-4d61-8260-5df9fcecd06a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:04:26.558Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f70ebed7-0f79-4d61-8260-5df9fcecd06a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:04:26.558Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f70ebed7-0f79-4d61-8260-5df9fcecd06a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:04:26.558Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f70ebed7-0f79-4d61-8260-5df9fcecd06a","podName":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:04:26.558Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f70ebed7-0f79-4d61-8260-5df9fcecd06a","podName":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:04:26.558Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f70ebed7-0f79-4d61-8260-5df9fcecd06a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:04:26.558Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f70ebed7-0f79-4d61-8260-5df9fcecd06a","seconds":300}
{"level":"info","ts":"2026-07-13T16:04:27.560Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"10805238-3971-4fde-aa08-caf2cf3e8e10"}
{"level":"info","ts":"2026-07-13T16:04:27.560Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"10805238-3971-4fde-aa08-caf2cf3e8e10"}
{"level":"info","ts":"2026-07-13T16:04:27.560Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"10805238-3971-4fde-aa08-caf2cf3e8e10","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:04:27.560Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"10805238-3971-4fde-aa08-caf2cf3e8e10","Found 1 head Pod":"fail-k8s-job-vrx7k-head-5fdmc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:04:27.560Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"10805238-3971-4fde-aa08-caf2cf3e8e10","head Pod":"fail-k8s-job-vrx7k-head-5fdmc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-vrx7k-head-5fdmc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:04:27.560Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"10805238-3971-4fde-aa08-caf2cf3e8e10","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:04:27.560Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"10805238-3971-4fde-aa08-caf2cf3e8e10","worker Pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-vrx7k-small-group-worker-lfkpv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:04:27.560Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"10805238-3971-4fde-aa08-caf2cf3e8e10","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:04:27.560Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"10805238-3971-4fde-aa08-caf2cf3e8e10","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:04:27.560Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"10805238-3971-4fde-aa08-caf2cf3e8e10","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:04:27.560Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"10805238-3971-4fde-aa08-caf2cf3e8e10","podName":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:04:27.560Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"10805238-3971-4fde-aa08-caf2cf3e8e10","podName":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:04:27.560Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"10805238-3971-4fde-aa08-caf2cf3e8e10","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:04:27.560Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"10805238-3971-4fde-aa08-caf2cf3e8e10","seconds":300}
{"level":"info","ts":"2026-07-13T16:04:28.306Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"6159e4f5-be12-4ce4-8d35-ef9351c799ec","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:28.306Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"6159e4f5-be12-4ce4-8d35-ef9351c799ec","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:28.306Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"6159e4f5-be12-4ce4-8d35-ef9351c799ec","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:28.306Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"6159e4f5-be12-4ce4-8d35-ef9351c799ec"}
{"level":"info","ts":"2026-07-13T16:04:28.306Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"6159e4f5-be12-4ce4-8d35-ef9351c799ec","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:04:31.307Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a85f3db5-bd9c-4407-9ac3-208d3a840efa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:31.307Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a85f3db5-bd9c-4407-9ac3-208d3a840efa","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:31.307Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a85f3db5-bd9c-4407-9ac3-208d3a840efa","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:31.307Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a85f3db5-bd9c-4407-9ac3-208d3a840efa"}
{"level":"info","ts":"2026-07-13T16:04:31.307Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a85f3db5-bd9c-4407-9ac3-208d3a840efa","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:04:34.308Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"9aafd769-0e79-44f7-80ab-5e7d75fd2fce","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:34.308Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"9aafd769-0e79-44f7-80ab-5e7d75fd2fce","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:34.308Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"9aafd769-0e79-44f7-80ab-5e7d75fd2fce","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:34.308Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"9aafd769-0e79-44f7-80ab-5e7d75fd2fce"}
{"level":"info","ts":"2026-07-13T16:04:34.308Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"9aafd769-0e79-44f7-80ab-5e7d75fd2fce","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:04:34.945Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"c1fe2233-e2e1-4847-b263-cc339e27570f","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:04:34.945Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"c1fe2233-e2e1-4847-b263-cc339e27570f","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T16:04:34.945Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"c1fe2233-e2e1-4847-b263-cc339e27570f","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T16:04:34.945Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"c1fe2233-e2e1-4847-b263-cc339e27570f","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T16:04:37.308Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"7fe07b49-8589-4ed4-b60b-d4cdf5553365","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:37.308Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"7fe07b49-8589-4ed4-b60b-d4cdf5553365","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:37.308Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"7fe07b49-8589-4ed4-b60b-d4cdf5553365","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:37.309Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"7fe07b49-8589-4ed4-b60b-d4cdf5553365"}
{"level":"info","ts":"2026-07-13T16:04:37.309Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"7fe07b49-8589-4ed4-b60b-d4cdf5553365","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:04:37.433Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"ff8e1fcc-6eb4-4e53-b426-8ff65d66fb13","rayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:37.433Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"ff8e1fcc-6eb4-4e53-b426-8ff65d66fb13","pod":"fail-k8s-job-vrx7k-head-5fdmc","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-13T16:04:37.433Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"ff8e1fcc-6eb4-4e53-b426-8ff65d66fb13","pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","ip":"10.134.0.33"}
{"level":"info","ts":"2026-07-13T16:04:37.433Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"ff8e1fcc-6eb4-4e53-b426-8ff65d66fb13","rayCluster":"fail-k8s-job-vrx7k","podIPs":["10.133.0.32","10.134.0.33"]}
{"level":"info","ts":"2026-07-13T16:04:37.609Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b545e25f-b2b9-43d8-84df-dd40280671cb"}
{"level":"info","ts":"2026-07-13T16:04:37.609Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b545e25f-b2b9-43d8-84df-dd40280671cb"}
{"level":"info","ts":"2026-07-13T16:04:37.609Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b545e25f-b2b9-43d8-84df-dd40280671cb","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:04:37.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b545e25f-b2b9-43d8-84df-dd40280671cb","Found 1 head Pod":"fail-k8s-job-vrx7k-head-5fdmc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:04:37.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b545e25f-b2b9-43d8-84df-dd40280671cb","head Pod":"fail-k8s-job-vrx7k-head-5fdmc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-vrx7k-head-5fdmc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:04:37.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b545e25f-b2b9-43d8-84df-dd40280671cb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:04:37.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b545e25f-b2b9-43d8-84df-dd40280671cb","worker Pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-vrx7k-small-group-worker-lfkpv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:04:37.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b545e25f-b2b9-43d8-84df-dd40280671cb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:04:37.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b545e25f-b2b9-43d8-84df-dd40280671cb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:04:37.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b545e25f-b2b9-43d8-84df-dd40280671cb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:04:37.609Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b545e25f-b2b9-43d8-84df-dd40280671cb","podName":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:04:37.609Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b545e25f-b2b9-43d8-84df-dd40280671cb","podName":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:04:37.609Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b545e25f-b2b9-43d8-84df-dd40280671cb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:04:37.609Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"b545e25f-b2b9-43d8-84df-dd40280671cb","seconds":300}
{"level":"info","ts":"2026-07-13T16:04:37.795Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db5b7616-4b00-4f67-8775-c40b24065b25"}
{"level":"info","ts":"2026-07-13T16:04:37.795Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db5b7616-4b00-4f67-8775-c40b24065b25"}
{"level":"info","ts":"2026-07-13T16:04:37.795Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db5b7616-4b00-4f67-8775-c40b24065b25","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:04:37.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db5b7616-4b00-4f67-8775-c40b24065b25","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:04:37.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db5b7616-4b00-4f67-8775-c40b24065b25","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:04:37.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db5b7616-4b00-4f67-8775-c40b24065b25","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:04:37.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db5b7616-4b00-4f67-8775-c40b24065b25","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:04:37.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db5b7616-4b00-4f67-8775-c40b24065b25","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:04:37.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db5b7616-4b00-4f67-8775-c40b24065b25","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:04:37.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db5b7616-4b00-4f67-8775-c40b24065b25","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:04:37.795Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db5b7616-4b00-4f67-8775-c40b24065b25","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:04:37.795Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"db5b7616-4b00-4f67-8775-c40b24065b25","seconds":300}
{"level":"info","ts":"2026-07-13T16:04:38.617Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1592af3a-7120-41b9-adad-331e39c373d3"}
{"level":"info","ts":"2026-07-13T16:04:38.617Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1592af3a-7120-41b9-adad-331e39c373d3"}
{"level":"info","ts":"2026-07-13T16:04:38.617Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1592af3a-7120-41b9-adad-331e39c373d3","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:04:38.617Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1592af3a-7120-41b9-adad-331e39c373d3","Found 1 head Pod":"fail-k8s-job-vrx7k-head-5fdmc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:04:38.617Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1592af3a-7120-41b9-adad-331e39c373d3","head Pod":"fail-k8s-job-vrx7k-head-5fdmc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-vrx7k-head-5fdmc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:04:38.617Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1592af3a-7120-41b9-adad-331e39c373d3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:04:38.617Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1592af3a-7120-41b9-adad-331e39c373d3","worker Pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-vrx7k-small-group-worker-lfkpv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:04:38.617Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1592af3a-7120-41b9-adad-331e39c373d3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:04:38.617Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1592af3a-7120-41b9-adad-331e39c373d3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:04:38.617Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1592af3a-7120-41b9-adad-331e39c373d3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:04:38.617Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1592af3a-7120-41b9-adad-331e39c373d3","podName":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:03:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:04:38.617Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1592af3a-7120-41b9-adad-331e39c373d3","podName":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:03:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:04:38.617Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1592af3a-7120-41b9-adad-331e39c373d3","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T16:04:38.617Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"1592af3a-7120-41b9-adad-331e39c373d3","name":"fail-k8s-job-vrx7k","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:03:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.32","serviceIP":"10.133.0.32","podName":"fail-k8s-job-vrx7k-head-5fdmc","serviceName":"fail-k8s-job-vrx7k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:03:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:04:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.32","serviceIP":"10.133.0.32","podName":"fail-k8s-job-vrx7k-head-5fdmc","serviceName":"fail-k8s-job-vrx7k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:03:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:04:38.630Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"1b7821ed-f8f6-4076-8418-5e56a7035628","rayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:38.630Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"3d04024f-fd69-4879-a7da-992d16fbdacc","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:38.630Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"1b7821ed-f8f6-4076-8418-5e56a7035628","pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","ip":"10.134.0.33"}
{"level":"info","ts":"2026-07-13T16:04:38.630Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"1b7821ed-f8f6-4076-8418-5e56a7035628","pod":"fail-k8s-job-vrx7k-head-5fdmc","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-13T16:04:38.630Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"1b7821ed-f8f6-4076-8418-5e56a7035628","rayCluster":"fail-k8s-job-vrx7k","podIPs":["10.134.0.33","10.133.0.32"]}
{"level":"info","ts":"2026-07-13T16:04:38.630Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b399e814-a94e-486b-8c07-514c36f0f60f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:38.630Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b399e814-a94e-486b-8c07-514c36f0f60f","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:38.630Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b399e814-a94e-486b-8c07-514c36f0f60f","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:38.630Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b399e814-a94e-486b-8c07-514c36f0f60f"}
{"level":"info","ts":"2026-07-13T16:04:38.630Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b399e814-a94e-486b-8c07-514c36f0f60f","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:04:38.645Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"3d04024f-fd69-4879-a7da-992d16fbdacc","name":"fail-k8s-job-vrx7k-head"}
{"level":"info","ts":"2026-07-13T16:04:38.655Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"3d04024f-fd69-4879-a7da-992d16fbdacc","name":"fail-k8s-job-vrx7k-workers"}
{"level":"info","ts":"2026-07-13T16:04:38.655Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"3d04024f-fd69-4879-a7da-992d16fbdacc","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:40.309Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b01a9dec-3e27-4538-adac-e8fc9bbc693e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:40.309Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b01a9dec-3e27-4538-adac-e8fc9bbc693e","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:40.309Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b01a9dec-3e27-4538-adac-e8fc9bbc693e","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:40.309Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b01a9dec-3e27-4538-adac-e8fc9bbc693e"}
{"level":"info","ts":"2026-07-13T16:04:40.309Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b01a9dec-3e27-4538-adac-e8fc9bbc693e","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:04:40.631Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f17bf034-28e5-4d2b-8f90-b33546c5e671"}
{"level":"info","ts":"2026-07-13T16:04:40.631Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f17bf034-28e5-4d2b-8f90-b33546c5e671"}
{"level":"info","ts":"2026-07-13T16:04:40.631Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f17bf034-28e5-4d2b-8f90-b33546c5e671","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:04:40.631Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f17bf034-28e5-4d2b-8f90-b33546c5e671","Found 1 head Pod":"fail-k8s-job-vrx7k-head-5fdmc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:04:40.631Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f17bf034-28e5-4d2b-8f90-b33546c5e671","head Pod":"fail-k8s-job-vrx7k-head-5fdmc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-vrx7k-head-5fdmc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:04:40.631Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f17bf034-28e5-4d2b-8f90-b33546c5e671","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:04:40.631Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f17bf034-28e5-4d2b-8f90-b33546c5e671","worker Pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-vrx7k-small-group-worker-lfkpv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:04:40.631Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f17bf034-28e5-4d2b-8f90-b33546c5e671","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:04:40.631Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f17bf034-28e5-4d2b-8f90-b33546c5e671","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:04:40.631Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f17bf034-28e5-4d2b-8f90-b33546c5e671","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:04:40.631Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f17bf034-28e5-4d2b-8f90-b33546c5e671","podName":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:03:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:04:40.631Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f17bf034-28e5-4d2b-8f90-b33546c5e671","podName":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:03:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:04:40.631Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f17bf034-28e5-4d2b-8f90-b33546c5e671","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:04:40.631Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"f17bf034-28e5-4d2b-8f90-b33546c5e671","seconds":300}
{"level":"info","ts":"2026-07-13T16:04:43.310Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1741e843-cf11-46ec-ac1f-5e606e7845ac","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:43.310Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1741e843-cf11-46ec-ac1f-5e606e7845ac","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:43.310Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1741e843-cf11-46ec-ac1f-5e606e7845ac","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:43.310Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1741e843-cf11-46ec-ac1f-5e606e7845ac"}
{"level":"info","ts":"2026-07-13T16:04:43.310Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1741e843-cf11-46ec-ac1f-5e606e7845ac","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:04:46.311Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"658ef145-1dae-4096-830c-844366502f02","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:46.311Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"658ef145-1dae-4096-830c-844366502f02","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:46.311Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"658ef145-1dae-4096-830c-844366502f02","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:46.311Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"658ef145-1dae-4096-830c-844366502f02"}
{"level":"info","ts":"2026-07-13T16:04:46.311Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"658ef145-1dae-4096-830c-844366502f02","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:04:49.312Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"011de851-8c97-496e-b4d5-69505396dd21","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:49.312Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"011de851-8c97-496e-b4d5-69505396dd21","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:49.312Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"011de851-8c97-496e-b4d5-69505396dd21","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:49.312Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"011de851-8c97-496e-b4d5-69505396dd21"}
{"level":"info","ts":"2026-07-13T16:04:49.312Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"011de851-8c97-496e-b4d5-69505396dd21","RayCluster":"fail-k8s-job-vrx7k","State":""}
{"level":"info","ts":"2026-07-13T16:04:49.640Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ea906ca0-5f89-46f9-8782-6e530d4b411d"}
{"level":"info","ts":"2026-07-13T16:04:49.640Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ea906ca0-5f89-46f9-8782-6e530d4b411d"}
{"level":"info","ts":"2026-07-13T16:04:49.640Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ea906ca0-5f89-46f9-8782-6e530d4b411d","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:04:49.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ea906ca0-5f89-46f9-8782-6e530d4b411d","Found 1 head Pod":"fail-k8s-job-vrx7k-head-5fdmc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:04:49.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ea906ca0-5f89-46f9-8782-6e530d4b411d","head Pod":"fail-k8s-job-vrx7k-head-5fdmc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-vrx7k-head-5fdmc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:04:49.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ea906ca0-5f89-46f9-8782-6e530d4b411d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:04:49.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ea906ca0-5f89-46f9-8782-6e530d4b411d","worker Pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-vrx7k-small-group-worker-lfkpv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:04:49.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ea906ca0-5f89-46f9-8782-6e530d4b411d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:04:49.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ea906ca0-5f89-46f9-8782-6e530d4b411d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:04:49.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ea906ca0-5f89-46f9-8782-6e530d4b411d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:04:49.640Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ea906ca0-5f89-46f9-8782-6e530d4b411d","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-13T16:04:49.640Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"ea906ca0-5f89-46f9-8782-6e530d4b411d","name":"fail-k8s-job-vrx7k","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:04:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.32","serviceIP":"10.133.0.32","podName":"fail-k8s-job-vrx7k-head-5fdmc","serviceName":"fail-k8s-job-vrx7k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:03:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:04:49Z","stateTransitionTimes":{"ready":"2026-07-13T16:04:49Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.32","serviceIP":"10.133.0.32","podName":"fail-k8s-job-vrx7k-head-5fdmc","serviceName":"fail-k8s-job-vrx7k-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:03:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:04:49Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:04:49.651Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"dc18bb1d-aa0d-41e6-a72e-ec8a0ac4e844","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:49.651Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"e2ce4aae-2dfb-4632-ab80-10fc052df32b","rayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:49.651Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"07e2d4bb-6d94-4fee-b73a-d9d126dd57dd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:49.651Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"07e2d4bb-6d94-4fee-b73a-d9d126dd57dd","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:49.651Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"07e2d4bb-6d94-4fee-b73a-d9d126dd57dd","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:49.651Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"e2ce4aae-2dfb-4632-ab80-10fc052df32b","pod":"fail-k8s-job-vrx7k-head-5fdmc","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-13T16:04:49.651Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"e2ce4aae-2dfb-4632-ab80-10fc052df32b","pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","ip":"10.134.0.33"}
{"level":"info","ts":"2026-07-13T16:04:49.652Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"e2ce4aae-2dfb-4632-ab80-10fc052df32b","rayCluster":"fail-k8s-job-vrx7k","podIPs":["10.133.0.32","10.134.0.33"]}
{"level":"info","ts":"2026-07-13T16:04:49.652Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"07e2d4bb-6d94-4fee-b73a-d9d126dd57dd"}
{"level":"info","ts":"2026-07-13T16:04:49.652Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"07e2d4bb-6d94-4fee-b73a-d9d126dd57dd","head service name":"fail-k8s-job-vrx7k-head-svc","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:04:49.652Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"07e2d4bb-6d94-4fee-b73a-d9d126dd57dd","head service URL":"fail-k8s-job-vrx7k-head-svc.test-ns-x5qvg.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-13T16:04:49.652Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"07e2d4bb-6d94-4fee-b73a-d9d126dd57dd"}
{"level":"info","ts":"2026-07-13T16:04:49.652Z","logger":"controllers.RayJob","msg":"User-provided command is used","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"07e2d4bb-6d94-4fee-b73a-d9d126dd57dd","command":["ray","job","submit","--address","http://do-not-exist:8265","--","echo 123"]}
{"level":"info","ts":"2026-07-13T16:04:49.659Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"07e2d4bb-6d94-4fee-b73a-d9d126dd57dd","Kubernetes Job":"fail-k8s-job"}
{"level":"info","ts":"2026-07-13T16:04:49.659Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"07e2d4bb-6d94-4fee-b73a-d9d126dd57dd","SubmissionMode":"K8sJobMode","RayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:49.659Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"07e2d4bb-6d94-4fee-b73a-d9d126dd57dd","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-qdx97","rayClusterName":"fail-k8s-job-vrx7k","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T16:03:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-qdx97","rayClusterName":"fail-k8s-job-vrx7k","dashboardURL":"fail-k8s-job-vrx7k-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T16:03:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T16:04:49.659Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"07e2d4bb-6d94-4fee-b73a-d9d126dd57dd","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T16:04:49.660Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"dc18bb1d-aa0d-41e6-a72e-ec8a0ac4e844","name":"fail-k8s-job-vrx7k-head"}
{"level":"info","ts":"2026-07-13T16:04:49.669Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"dc18bb1d-aa0d-41e6-a72e-ec8a0ac4e844","name":"fail-k8s-job-vrx7k-workers"}
{"level":"info","ts":"2026-07-13T16:04:49.669Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"dc18bb1d-aa0d-41e6-a72e-ec8a0ac4e844","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:49.671Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"7514db64-c8c6-4a60-ab22-db5aecd882de","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:49.671Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"7514db64-c8c6-4a60-ab22-db5aecd882de","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:49.671Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"7514db64-c8c6-4a60-ab22-db5aecd882de","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:49.671Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"7514db64-c8c6-4a60-ab22-db5aecd882de"}
{"level":"error","ts":"2026-07-13T16:04:49.725Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"7514db64-c8c6-4a60-ab22-db5aecd882de","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:49.726Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"7514db64-c8c6-4a60-ab22-db5aecd882de"}
{"level":"error","ts":"2026-07-13T16:04:49.726Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"7514db64-c8c6-4a60-ab22-db5aecd882de","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:49.726Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"fde25acc-c1b7-4fd3-9e20-e03d3c32100a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:49.726Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"fde25acc-c1b7-4fd3-9e20-e03d3c32100a","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:49.726Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"fde25acc-c1b7-4fd3-9e20-e03d3c32100a","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:49.726Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"fde25acc-c1b7-4fd3-9e20-e03d3c32100a"}
{"level":"error","ts":"2026-07-13T16:04:49.730Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"fde25acc-c1b7-4fd3-9e20-e03d3c32100a","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:49.730Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"fde25acc-c1b7-4fd3-9e20-e03d3c32100a"}
{"level":"error","ts":"2026-07-13T16:04:49.730Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"fde25acc-c1b7-4fd3-9e20-e03d3c32100a","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:49.731Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"06043d71-ba2b-4f2e-ab2e-913236b2b7c2","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:49.731Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"06043d71-ba2b-4f2e-ab2e-913236b2b7c2","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:49.731Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"06043d71-ba2b-4f2e-ab2e-913236b2b7c2","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:49.731Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"06043d71-ba2b-4f2e-ab2e-913236b2b7c2"}
{"level":"error","ts":"2026-07-13T16:04:49.735Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"06043d71-ba2b-4f2e-ab2e-913236b2b7c2","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:49.735Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"06043d71-ba2b-4f2e-ab2e-913236b2b7c2"}
{"level":"error","ts":"2026-07-13T16:04:49.735Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"06043d71-ba2b-4f2e-ab2e-913236b2b7c2","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:49.756Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"4a0fbe09-b6d7-4e1a-bb6c-5fa084549a3c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:49.756Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"4a0fbe09-b6d7-4e1a-bb6c-5fa084549a3c","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:49.756Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"4a0fbe09-b6d7-4e1a-bb6c-5fa084549a3c","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:49.756Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"4a0fbe09-b6d7-4e1a-bb6c-5fa084549a3c"}
{"level":"error","ts":"2026-07-13T16:04:49.760Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"4a0fbe09-b6d7-4e1a-bb6c-5fa084549a3c","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:49.760Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"4a0fbe09-b6d7-4e1a-bb6c-5fa084549a3c"}
{"level":"error","ts":"2026-07-13T16:04:49.760Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"4a0fbe09-b6d7-4e1a-bb6c-5fa084549a3c","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:49.800Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"e6d3c2ee-5e4b-44fb-8431-85fe94b796f3","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:49.800Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"e6d3c2ee-5e4b-44fb-8431-85fe94b796f3","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:49.800Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"e6d3c2ee-5e4b-44fb-8431-85fe94b796f3","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:49.800Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"e6d3c2ee-5e4b-44fb-8431-85fe94b796f3"}
{"level":"error","ts":"2026-07-13T16:04:49.804Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"e6d3c2ee-5e4b-44fb-8431-85fe94b796f3","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:49.804Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"e6d3c2ee-5e4b-44fb-8431-85fe94b796f3"}
{"level":"error","ts":"2026-07-13T16:04:49.804Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"e6d3c2ee-5e4b-44fb-8431-85fe94b796f3","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:49.885Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"563a6826-9d0c-434b-818a-8df510ac26cf","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:49.885Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"563a6826-9d0c-434b-818a-8df510ac26cf","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:49.885Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"563a6826-9d0c-434b-818a-8df510ac26cf","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:49.885Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"563a6826-9d0c-434b-818a-8df510ac26cf"}
{"level":"error","ts":"2026-07-13T16:04:49.889Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"563a6826-9d0c-434b-818a-8df510ac26cf","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:49.889Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"563a6826-9d0c-434b-818a-8df510ac26cf"}
{"level":"error","ts":"2026-07-13T16:04:49.889Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"563a6826-9d0c-434b-818a-8df510ac26cf","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:50.050Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"5c35d1fc-eb3c-4089-a5bf-4224e066fd0b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:50.050Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"5c35d1fc-eb3c-4089-a5bf-4224e066fd0b","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:50.050Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"5c35d1fc-eb3c-4089-a5bf-4224e066fd0b","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:50.050Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"5c35d1fc-eb3c-4089-a5bf-4224e066fd0b"}
{"level":"error","ts":"2026-07-13T16:04:50.056Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"5c35d1fc-eb3c-4089-a5bf-4224e066fd0b","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:50.056Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"5c35d1fc-eb3c-4089-a5bf-4224e066fd0b"}
{"level":"error","ts":"2026-07-13T16:04:50.056Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"5c35d1fc-eb3c-4089-a5bf-4224e066fd0b","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:50.377Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"f90d26ba-33a7-466f-93a9-42fe5e782612","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:50.377Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"f90d26ba-33a7-466f-93a9-42fe5e782612","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:50.377Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"f90d26ba-33a7-466f-93a9-42fe5e782612","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:50.378Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"f90d26ba-33a7-466f-93a9-42fe5e782612"}
{"level":"error","ts":"2026-07-13T16:04:50.383Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"f90d26ba-33a7-466f-93a9-42fe5e782612","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:50.383Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"f90d26ba-33a7-466f-93a9-42fe5e782612"}
{"level":"error","ts":"2026-07-13T16:04:50.383Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"f90d26ba-33a7-466f-93a9-42fe5e782612","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:50.682Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0da39841-ede2-462f-bbda-24018207b920","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:50.682Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0da39841-ede2-462f-bbda-24018207b920","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:50.682Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0da39841-ede2-462f-bbda-24018207b920","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:50.683Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0da39841-ede2-462f-bbda-24018207b920"}
{"level":"error","ts":"2026-07-13T16:04:50.687Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0da39841-ede2-462f-bbda-24018207b920","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:50.687Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0da39841-ede2-462f-bbda-24018207b920"}
{"level":"error","ts":"2026-07-13T16:04:50.687Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"0da39841-ede2-462f-bbda-24018207b920","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:51.023Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"6cf37d84-7fe9-4a55-b50c-bc923810d34a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:51.023Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"6cf37d84-7fe9-4a55-b50c-bc923810d34a","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:51.023Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"6cf37d84-7fe9-4a55-b50c-bc923810d34a","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:51.023Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"6cf37d84-7fe9-4a55-b50c-bc923810d34a"}
{"level":"error","ts":"2026-07-13T16:04:51.028Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"6cf37d84-7fe9-4a55-b50c-bc923810d34a","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:51.028Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"6cf37d84-7fe9-4a55-b50c-bc923810d34a"}
{"level":"error","ts":"2026-07-13T16:04:51.028Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"6cf37d84-7fe9-4a55-b50c-bc923810d34a","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:51.651Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"541a57b0-bfa1-4d84-b963-00b70abfbcf7"}
{"level":"info","ts":"2026-07-13T16:04:51.651Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"541a57b0-bfa1-4d84-b963-00b70abfbcf7"}
{"level":"info","ts":"2026-07-13T16:04:51.651Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"541a57b0-bfa1-4d84-b963-00b70abfbcf7","1 head service found":"fail-k8s-job-vrx7k-head-svc"}
{"level":"info","ts":"2026-07-13T16:04:51.651Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"541a57b0-bfa1-4d84-b963-00b70abfbcf7","Found 1 head Pod":"fail-k8s-job-vrx7k-head-5fdmc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:04:51.651Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"541a57b0-bfa1-4d84-b963-00b70abfbcf7","head Pod":"fail-k8s-job-vrx7k-head-5fdmc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-vrx7k-head-5fdmc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:04:51.651Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"541a57b0-bfa1-4d84-b963-00b70abfbcf7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:04:51.651Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"541a57b0-bfa1-4d84-b963-00b70abfbcf7","worker Pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-vrx7k-small-group-worker-lfkpv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:04:51.651Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"541a57b0-bfa1-4d84-b963-00b70abfbcf7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:04:51.651Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"541a57b0-bfa1-4d84-b963-00b70abfbcf7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:04:51.651Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"541a57b0-bfa1-4d84-b963-00b70abfbcf7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:04:51.652Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"541a57b0-bfa1-4d84-b963-00b70abfbcf7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:04:51.652Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"541a57b0-bfa1-4d84-b963-00b70abfbcf7","seconds":300}
{"level":"info","ts":"2026-07-13T16:04:53.589Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"44a1b278-d043-4afb-8e75-86f351607c7d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:53.589Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"44a1b278-d043-4afb-8e75-86f351607c7d","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:53.589Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"44a1b278-d043-4afb-8e75-86f351607c7d","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:53.589Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"44a1b278-d043-4afb-8e75-86f351607c7d"}
{"level":"error","ts":"2026-07-13T16:04:53.594Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"44a1b278-d043-4afb-8e75-86f351607c7d","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:53.594Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"44a1b278-d043-4afb-8e75-86f351607c7d"}
{"level":"error","ts":"2026-07-13T16:04:53.594Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"44a1b278-d043-4afb-8e75-86f351607c7d","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:55.329Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d8cb6102-f9f7-4860-bda0-7faf46e58039","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:55.329Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d8cb6102-f9f7-4860-bda0-7faf46e58039","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:55.329Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d8cb6102-f9f7-4860-bda0-7faf46e58039","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:55.329Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d8cb6102-f9f7-4860-bda0-7faf46e58039"}
{"level":"error","ts":"2026-07-13T16:04:55.335Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d8cb6102-f9f7-4860-bda0-7faf46e58039","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:55.335Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d8cb6102-f9f7-4860-bda0-7faf46e58039"}
{"level":"error","ts":"2026-07-13T16:04:55.335Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"d8cb6102-f9f7-4860-bda0-7faf46e58039","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:56.335Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c16b55d1-ffca-4982-ba34-d21d106ca932","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:56.335Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c16b55d1-ffca-4982-ba34-d21d106ca932","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:56.335Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c16b55d1-ffca-4982-ba34-d21d106ca932","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:56.335Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c16b55d1-ffca-4982-ba34-d21d106ca932"}
{"level":"error","ts":"2026-07-13T16:04:56.339Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c16b55d1-ffca-4982-ba34-d21d106ca932","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:56.340Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c16b55d1-ffca-4982-ba34-d21d106ca932"}
{"level":"error","ts":"2026-07-13T16:04:56.340Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c16b55d1-ffca-4982-ba34-d21d106ca932","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:56.350Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1587a028-8c0a-4741-9dc6-51598bd40833","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:56.350Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1587a028-8c0a-4741-9dc6-51598bd40833","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:56.350Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1587a028-8c0a-4741-9dc6-51598bd40833","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:56.350Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1587a028-8c0a-4741-9dc6-51598bd40833"}
{"level":"error","ts":"2026-07-13T16:04:56.354Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1587a028-8c0a-4741-9dc6-51598bd40833","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:56.354Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1587a028-8c0a-4741-9dc6-51598bd40833"}
{"level":"error","ts":"2026-07-13T16:04:56.354Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1587a028-8c0a-4741-9dc6-51598bd40833","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:58.715Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"379908e9-6352-49e3-aef3-195acf036c70","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:04:58.715Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"379908e9-6352-49e3-aef3-195acf036c70","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:04:58.715Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"379908e9-6352-49e3-aef3-195acf036c70","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:04:58.715Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"379908e9-6352-49e3-aef3-195acf036c70"}
{"level":"error","ts":"2026-07-13T16:04:58.720Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"379908e9-6352-49e3-aef3-195acf036c70","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:04:58.720Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"379908e9-6352-49e3-aef3-195acf036c70"}
{"level":"error","ts":"2026-07-13T16:04:58.720Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"379908e9-6352-49e3-aef3-195acf036c70","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:03.021Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a9e59efb-3729-44eb-a2f7-c063c8e0f7f6","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:03.021Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a9e59efb-3729-44eb-a2f7-c063c8e0f7f6","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:03.021Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a9e59efb-3729-44eb-a2f7-c063c8e0f7f6","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:03.021Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a9e59efb-3729-44eb-a2f7-c063c8e0f7f6"}
{"level":"error","ts":"2026-07-13T16:05:03.026Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a9e59efb-3729-44eb-a2f7-c063c8e0f7f6","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:03.026Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a9e59efb-3729-44eb-a2f7-c063c8e0f7f6"}
{"level":"error","ts":"2026-07-13T16:05:03.026Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"a9e59efb-3729-44eb-a2f7-c063c8e0f7f6","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:05.376Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"f7e589ee-14ba-4824-8ca2-a8889f84e715","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:05.376Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"f7e589ee-14ba-4824-8ca2-a8889f84e715","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:05.376Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"f7e589ee-14ba-4824-8ca2-a8889f84e715","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:05.376Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"f7e589ee-14ba-4824-8ca2-a8889f84e715"}
{"level":"error","ts":"2026-07-13T16:05:05.382Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"f7e589ee-14ba-4824-8ca2-a8889f84e715","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:05.382Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"f7e589ee-14ba-4824-8ca2-a8889f84e715"}
{"level":"error","ts":"2026-07-13T16:05:05.382Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"f7e589ee-14ba-4824-8ca2-a8889f84e715","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:08.391Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c844277b-ef43-4b16-a684-6845b26f2d32","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:08.391Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c844277b-ef43-4b16-a684-6845b26f2d32","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:08.391Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c844277b-ef43-4b16-a684-6845b26f2d32","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:08.391Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c844277b-ef43-4b16-a684-6845b26f2d32"}
{"level":"error","ts":"2026-07-13T16:05:08.397Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c844277b-ef43-4b16-a684-6845b26f2d32","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:08.397Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c844277b-ef43-4b16-a684-6845b26f2d32"}
{"level":"error","ts":"2026-07-13T16:05:08.397Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"c844277b-ef43-4b16-a684-6845b26f2d32","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:09.398Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"6da54a41-fea7-445a-8a2d-07488092d932","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:09.398Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"6da54a41-fea7-445a-8a2d-07488092d932","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:09.398Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"6da54a41-fea7-445a-8a2d-07488092d932","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:09.398Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"6da54a41-fea7-445a-8a2d-07488092d932"}
{"level":"error","ts":"2026-07-13T16:05:09.403Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"6da54a41-fea7-445a-8a2d-07488092d932","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:09.403Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"6da54a41-fea7-445a-8a2d-07488092d932"}
{"level":"error","ts":"2026-07-13T16:05:09.403Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"6da54a41-fea7-445a-8a2d-07488092d932","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:09.415Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1113ab19-6ef9-47bf-8e3c-f1bb2d04f187","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:09.415Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1113ab19-6ef9-47bf-8e3c-f1bb2d04f187","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:09.415Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1113ab19-6ef9-47bf-8e3c-f1bb2d04f187","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:09.415Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1113ab19-6ef9-47bf-8e3c-f1bb2d04f187"}
{"level":"error","ts":"2026-07-13T16:05:09.420Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1113ab19-6ef9-47bf-8e3c-f1bb2d04f187","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:09.420Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1113ab19-6ef9-47bf-8e3c-f1bb2d04f187"}
{"level":"error","ts":"2026-07-13T16:05:09.420Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"1113ab19-6ef9-47bf-8e3c-f1bb2d04f187","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:27.022Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"75cd9fb4-fa05-4d5d-befd-5c4456904a69","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:27.022Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"75cd9fb4-fa05-4d5d-befd-5c4456904a69","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:27.022Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"75cd9fb4-fa05-4d5d-befd-5c4456904a69","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:27.022Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"75cd9fb4-fa05-4d5d-befd-5c4456904a69"}
{"level":"error","ts":"2026-07-13T16:05:27.029Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"75cd9fb4-fa05-4d5d-befd-5c4456904a69","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:27.030Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"75cd9fb4-fa05-4d5d-befd-5c4456904a69"}
{"level":"error","ts":"2026-07-13T16:05:27.030Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"75cd9fb4-fa05-4d5d-befd-5c4456904a69","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:29.475Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ebcd7975-5613-436d-9f22-4e00056d7b56","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:29.475Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ebcd7975-5613-436d-9f22-4e00056d7b56","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:29.475Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ebcd7975-5613-436d-9f22-4e00056d7b56","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:29.475Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ebcd7975-5613-436d-9f22-4e00056d7b56"}
{"level":"error","ts":"2026-07-13T16:05:29.480Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ebcd7975-5613-436d-9f22-4e00056d7b56","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:29.480Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ebcd7975-5613-436d-9f22-4e00056d7b56"}
{"level":"error","ts":"2026-07-13T16:05:29.480Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ebcd7975-5613-436d-9f22-4e00056d7b56","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:32.489Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"48049f60-bc4b-4c51-bcf3-f893d00c4435","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:32.489Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"48049f60-bc4b-4c51-bcf3-f893d00c4435","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:32.489Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"48049f60-bc4b-4c51-bcf3-f893d00c4435","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:32.490Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"48049f60-bc4b-4c51-bcf3-f893d00c4435"}
{"level":"error","ts":"2026-07-13T16:05:32.495Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"48049f60-bc4b-4c51-bcf3-f893d00c4435","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:32.495Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"48049f60-bc4b-4c51-bcf3-f893d00c4435"}
{"level":"error","ts":"2026-07-13T16:05:32.495Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"48049f60-bc4b-4c51-bcf3-f893d00c4435","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:33.497Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b4c02923-9a7c-4633-8efc-bd44a5ed38e8","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:33.497Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b4c02923-9a7c-4633-8efc-bd44a5ed38e8","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:33.497Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b4c02923-9a7c-4633-8efc-bd44a5ed38e8","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:33.497Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b4c02923-9a7c-4633-8efc-bd44a5ed38e8"}
{"level":"error","ts":"2026-07-13T16:05:33.502Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b4c02923-9a7c-4633-8efc-bd44a5ed38e8","JobId":"fail-k8s-job-qdx97","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:33.502Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b4c02923-9a7c-4633-8efc-bd44a5ed38e8"}
{"level":"error","ts":"2026-07-13T16:05:33.502Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b4c02923-9a7c-4633-8efc-bd44a5ed38e8","error":"Job fail-k8s-job-qdx97 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:33.514Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b430c726-de64-46f6-8867-53ff35a66f99","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:33.514Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has failed. Attempting to transition the status to `Failed`.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b430c726-de64-46f6-8867-53ff35a66f99","Submitter K8s Job":"fail-k8s-job","Reason":"BackoffLimitExceeded","Message":"Job has reached the specified backoff limit"}
{"level":"info","ts":"2026-07-13T16:05:33.514Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b430c726-de64-46f6-8867-53ff35a66f99","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-qdx97","rayClusterName":"fail-k8s-job-vrx7k","dashboardURL":"fail-k8s-job-vrx7k-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T16:03:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-qdx97","rayClusterName":"fail-k8s-job-vrx7k","dashboardURL":"fail-k8s-job-vrx7k-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobDeploymentStatus":"Failed","reason":"SubmissionFailed","message":"Job submission has failed. Reason: BackoffLimitExceeded. Message: Job has reached the specified backoff limit","startTime":"2026-07-13T16:03:07Z","succeeded":0,"failed":1,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T16:05:33.514Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"b430c726-de64-46f6-8867-53ff35a66f99","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-07-13T16:05:33.527Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"5a5a3a2b-81e6-4860-a023-b9bf381b5e21","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:33.527Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"5a5a3a2b-81e6-4860-a023-b9bf381b5e21","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 16:05:33 +0000 UTC","Now":"2026-07-13T16:05:33.527Z","ShutdownTime":"2026-07-13T16:05:33.000Z"}
{"level":"info","ts":"2026-07-13T16:05:33.527Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"5a5a3a2b-81e6-4860-a023-b9bf381b5e21","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T16:05:33.535Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"5a5a3a2b-81e6-4860-a023-b9bf381b5e21","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:33.536Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"5a5a3a2b-81e6-4860-a023-b9bf381b5e21","isClusterDeleted":false}
{"level":"info","ts":"2026-07-13T16:05:33.536Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"5a5a3a2b-81e6-4860-a023-b9bf381b5e21","RayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:33.536Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"820ce60c-e5fc-4e75-8628-230ae082322a","namespacedName":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:33.536Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"820ce60c-e5fc-4e75-8628-230ae082322a","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:33.536Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"820ce60c-e5fc-4e75-8628-230ae082322a","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:33.536Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"f3c118c5-e74d-4496-bb82-951b47f0809a","rayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:33.536Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"9c73f2a5-5a46-467f-a25f-109da90248d1"}
{"level":"info","ts":"2026-07-13T16:05:33.536Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"820ce60c-e5fc-4e75-8628-230ae082322a","namespace":"test-ns-x5qvg","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-13T16:05:33.536Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"reconcileID":"c772bf08-f347-4ed3-a0e5-e314eb4917f2"}
{"level":"info","ts":"2026-07-13T16:05:33.536Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"12489c0c-8859-4e62-8522-57a238795898","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:33.536Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"f3c118c5-e74d-4496-bb82-951b47f0809a","pod":"fail-k8s-job-vrx7k-head-5fdmc","ip":"10.133.0.32"}
{"level":"info","ts":"2026-07-13T16:05:33.536Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"f3c118c5-e74d-4496-bb82-951b47f0809a","pod":"fail-k8s-job-vrx7k-small-group-worker-lfkpv","ip":"10.134.0.33"}
{"level":"info","ts":"2026-07-13T16:05:33.536Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"12489c0c-8859-4e62-8522-57a238795898","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 16:05:33 +0000 UTC","Now":"2026-07-13T16:05:33.536Z","ShutdownTime":"2026-07-13T16:05:33.000Z"}
{"level":"info","ts":"2026-07-13T16:05:33.536Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"12489c0c-8859-4e62-8522-57a238795898","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T16:05:33.536Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"12489c0c-8859-4e62-8522-57a238795898","RayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:33.536Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"12489c0c-8859-4e62-8522-57a238795898","isClusterDeleted":false}
{"level":"info","ts":"2026-07-13T16:05:33.536Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"12489c0c-8859-4e62-8522-57a238795898","RayCluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:33.536Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"f3c118c5-e74d-4496-bb82-951b47f0809a","rayCluster":"fail-k8s-job-vrx7k","podIPs":["10.133.0.32","10.134.0.33"]}
{"level":"info","ts":"2026-07-13T16:05:33.542Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"820ce60c-e5fc-4e75-8628-230ae082322a","name":"kuberay-gateway-access","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:05:33.547Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"820ce60c-e5fc-4e75-8628-230ae082322a","name":"test-ns-x5qvg-fail-k8s-job-vrx7k","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-13T16:05:33.552Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"820ce60c-e5fc-4e75-8628-230ae082322a","configMap":"kube-rbac-proxy-config-fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:33.556Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"820ce60c-e5fc-4e75-8628-230ae082322a","serviceAccount":"fail-k8s-job-vrx7k-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:05:33.574Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:33.574Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:05:33.583Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"820ce60c-e5fc-4e75-8628-230ae082322a","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:33.583Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"d03aa43f-75ae-4c35-803b-b1af48225e7a","namespacedName":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:33.583Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"d03aa43f-75ae-4c35-803b-b1af48225e7a","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:33.583Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"d03aa43f-75ae-4c35-803b-b1af48225e7a","cluster":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:33.583Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"d03aa43f-75ae-4c35-803b-b1af48225e7a","namespace":"test-ns-x5qvg","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-13T16:05:33.583Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"d03aa43f-75ae-4c35-803b-b1af48225e7a","name":"kuberay-gateway-access"}
{"level":"error","ts":"2026-07-13T16:05:33.598Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"d03aa43f-75ae-4c35-803b-b1af48225e7a","error":"rayclusters.ray.io \"fail-k8s-job-vrx7k\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:33.598Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"d03aa43f-75ae-4c35-803b-b1af48225e7a"}
{"level":"error","ts":"2026-07-13T16:05:33.598Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"d03aa43f-75ae-4c35-803b-b1af48225e7a","error":"rayclusters.ray.io \"fail-k8s-job-vrx7k\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:33.598Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"9ba7f543-ef3e-4fed-b433-111cb2319953"}
{"level":"info","ts":"2026-07-13T16:05:33.621Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"81512be3-dfff-4d29-9f9b-7419400bb5ae","namespacedName":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:33.621Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"81512be3-dfff-4d29-9f9b-7419400bb5ae"}
{"level":"info","ts":"2026-07-13T16:05:33.621Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"558dafa8-6d1f-4044-8bcb-c87fd36c2b5a"}
{"level":"info","ts":"2026-07-13T16:05:33.621Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"81512be3-dfff-4d29-9f9b-7419400bb5ae","namespace":"test-ns-x5qvg","authClusters":0}
{"level":"info","ts":"2026-07-13T16:05:33.621Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"81512be3-dfff-4d29-9f9b-7419400bb5ae","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:05:33.621Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"874a7817-30c7-48b3-9ce4-01a8fef73a20"}
{"level":"info","ts":"2026-07-13T16:05:33.622Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"019c18d3-f52a-4c20-affb-0e7ce526e822","namespacedName":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:33.622Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"019c18d3-f52a-4c20-affb-0e7ce526e822"}
{"level":"info","ts":"2026-07-13T16:05:33.622Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"019c18d3-f52a-4c20-affb-0e7ce526e822","namespace":"test-ns-x5qvg","authClusters":0}
{"level":"info","ts":"2026-07-13T16:05:33.621Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"476032ed-1392-4071-9c41-adfe7c19ef2b","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:33.622Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"019c18d3-f52a-4c20-affb-0e7ce526e822","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:05:33.621Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"afec3c0f-0823-46c3-8e1c-742c00146767","namespace":"test-ns-x5qvg","clusterName":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:33.622Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"476032ed-1392-4071-9c41-adfe7c19ef2b","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 16:05:33 +0000 UTC","Now":"2026-07-13T16:05:33.622Z","ShutdownTime":"2026-07-13T16:05:33.000Z"}
{"level":"info","ts":"2026-07-13T16:05:33.622Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"476032ed-1392-4071-9c41-adfe7c19ef2b","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T16:05:33.622Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"476032ed-1392-4071-9c41-adfe7c19ef2b","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:33.622Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"476032ed-1392-4071-9c41-adfe7c19ef2b","isClusterDeleted":true}
{"level":"info","ts":"2026-07-13T16:05:33.622Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"476032ed-1392-4071-9c41-adfe7c19ef2b","RayCluster":"fail-k8s-job-vrx7k"}
{"level":"error","ts":"2026-07-13T16:05:33.627Z","msg":"Failed to delete CA issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"afec3c0f-0823-46c3-8e1c-742c00146767","issuer":"ray-ca-issuer-fail-k8s-job-vrx7k","error":"issuers.cert-manager.io \"ray-ca-issuer-fail-k8s-job-vrx7k\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:247\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:33.642Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"afec3c0f-0823-46c3-8e1c-742c00146767","errorCount":1}
{"level":"info","ts":"2026-07-13T16:05:33.642Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"afec3c0f-0823-46c3-8e1c-742c00146767"}
{"level":"error","ts":"2026-07-13T16:05:33.642Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"afec3c0f-0823-46c3-8e1c-742c00146767","error":"issuers.cert-manager.io \"ray-ca-issuer-fail-k8s-job-vrx7k\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:33.648Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"17b2c498-d7eb-44d3-8fa5-f639da5c9a62","namespace":"test-ns-x5qvg","clusterName":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:33.648Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-vrx7k","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"fail-k8s-job-vrx7k","reconcileID":"17b2c498-d7eb-44d3-8fa5-f639da5c9a62","namespace":"test-ns-x5qvg","clusterName":"fail-k8s-job-vrx7k"}
{"level":"info","ts":"2026-07-13T16:05:34.035Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"9dca3691-56ca-4382-97c4-a562718831a6","DeletionTimestamp":"2026-07-13 16:05:34 +0000 UTC"}
{"level":"error","ts":"2026-07-13T16:05:34.035Z","logger":"controllers.RayJob","msg":"Failed to get RayCluster","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"9dca3691-56ca-4382-97c4-a562718831a6","error":"RayCluster.ray.io \"fail-k8s-job-vrx7k\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:115\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:34.035Z","logger":"controllers.RayJob","msg":"Stop a ray job","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"9dca3691-56ca-4382-97c4-a562718831a6","rayJob":"fail-k8s-job-qdx97"}
{"level":"error","ts":"2026-07-13T16:05:34.037Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"9dca3691-56ca-4382-97c4-a562718831a6","error":"Post \"http://fail-k8s-job-vrx7k-head-svc.test-ns-x5qvg.svc.cluster.local:8265/api/jobs/fail-k8s-job-qdx97/stop\": dial tcp 10.133.0.32:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:123\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:34.037Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"9dca3691-56ca-4382-97c4-a562718831a6","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T16:05:34.048Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"900d1280-e1a8-4a25-9ec1-1b982b019b8e","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:34.048Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"900d1280-e1a8-4a25-9ec1-1b982b019b8e","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T16:05:34.059Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"900d1280-e1a8-4a25-9ec1-1b982b019b8e"}
{"level":"info","ts":"2026-07-13T16:05:34.059Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"900d1280-e1a8-4a25-9ec1-1b982b019b8e","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-13T16:05:34.059Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"900d1280-e1a8-4a25-9ec1-1b982b019b8e","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T16:05:34.059Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"900d1280-e1a8-4a25-9ec1-1b982b019b8e","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T16:05:34.072Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"ec601111-9a1b-4ff8-ae8f-dcd9c568461c"}
{"level":"info","ts":"2026-07-13T16:05:34.072Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6d1d3d4c-093c-4a91-8d6f-ea8655f8a6e7","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:34.072Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6d1d3d4c-093c-4a91-8d6f-ea8655f8a6e7"}
{"level":"info","ts":"2026-07-13T16:05:34.072Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6d1d3d4c-093c-4a91-8d6f-ea8655f8a6e7","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-13T16:05:34.072Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6d1d3d4c-093c-4a91-8d6f-ea8655f8a6e7","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-zwstw","rayClusterName":"stop-49ztr","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T16:05:34.072Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6d1d3d4c-093c-4a91-8d6f-ea8655f8a6e7","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T16:05:34.078Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6d1d3d4c-093c-4a91-8d6f-ea8655f8a6e7","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-07-13T16:05:34.078Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6d1d3d4c-093c-4a91-8d6f-ea8655f8a6e7"}
{"level":"error","ts":"2026-07-13T16:05:34.078Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6d1d3d4c-093c-4a91-8d6f-ea8655f8a6e7","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:34.078Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ae6e283d-df82-4c60-8c86-8599ff527270","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:34.078Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ae6e283d-df82-4c60-8c86-8599ff527270","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.078Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ae6e283d-df82-4c60-8c86-8599ff527270","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:34.078Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ae6e283d-df82-4c60-8c86-8599ff527270","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:34.082Z","logger":"raycluster-resource","msg":"default","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.082Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-b592q","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:05:34.096Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ae6e283d-df82-4c60-8c86-8599ff527270","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:34.096Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ae6e283d-df82-4c60-8c86-8599ff527270"}
{"level":"info","ts":"2026-07-13T16:05:34.096Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ae6e283d-df82-4c60-8c86-8599ff527270","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:05:34.121Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"e49ef3ab-aa5d-46f4-9546-f7918d92b331","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:34.121Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"e49ef3ab-aa5d-46f4-9546-f7918d92b331","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.121Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"e49ef3ab-aa5d-46f4-9546-f7918d92b331","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:34.121Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"e49ef3ab-aa5d-46f4-9546-f7918d92b331","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:34.121Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"6be0f9be-35bd-4809-bcf4-e4f00f775518","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.121Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"f36a45ca-f400-4e46-84bf-9fdebef0a79b","rayCluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.121Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"f36a45ca-f400-4e46-84bf-9fdebef0a79b","rayCluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.121Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"2d93c048-1f94-4f40-ad82-f691c223cb65"}
{"level":"info","ts":"2026-07-13T16:05:34.121Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"2d93c048-1f94-4f40-ad82-f691c223cb65"}
{"level":"info","ts":"2026-07-13T16:05:34.121Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"7afe5749-b370-471c-994d-b02435dca24a","namespacedName":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:34.121Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"7afe5749-b370-471c-994d-b02435dca24a","mode":"IntegratedOAuth","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.122Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"7afe5749-b370-471c-994d-b02435dca24a","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.122Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"7afe5749-b370-471c-994d-b02435dca24a","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.125Z","logger":"raycluster-resource","msg":"default","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.125Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-b592q","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:05:34.128Z","logger":"raycluster-resource","msg":"default","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.128Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-b592q","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:05:34.131Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"2d93c048-1f94-4f40-ad82-f691c223cb65","name":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:05:34.131Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"2d93c048-1f94-4f40-ad82-f691c223cb65","error":"head certificate secret ray-head-secret-stop-b592q not found"}
{"level":"error","ts":"2026-07-13T16:05:34.131Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"2d93c048-1f94-4f40-ad82-f691c223cb65","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:34.131Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"2d93c048-1f94-4f40-ad82-f691c223cb65","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found"}
{"level":"info","ts":"2026-07-13T16:05:34.221Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"2d93c048-1f94-4f40-ad82-f691c223cb65","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:34.221Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"2d93c048-1f94-4f40-ad82-f691c223cb65","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:34.221Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"2d93c048-1f94-4f40-ad82-f691c223cb65","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:34.221Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"2d93c048-1f94-4f40-ad82-f691c223cb65","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T16:05:34.221Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"2d93c048-1f94-4f40-ad82-f691c223cb65","name":"stop-b592q","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:05:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:05:34.221Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"e49ef3ab-aa5d-46f4-9546-f7918d92b331"}
{"level":"error","ts":"2026-07-13T16:05:34.221Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"e49ef3ab-aa5d-46f4-9546-f7918d92b331","error":"rayclusters.ray.io \"stop-b592q\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:34.221Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"dcc5e4d1-0c92-4712-af89-1185f5c2b703","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:34.221Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"dcc5e4d1-0c92-4712-af89-1185f5c2b703","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.221Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"dcc5e4d1-0c92-4712-af89-1185f5c2b703","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:34.222Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"f36a45ca-f400-4e46-84bf-9fdebef0a79b","rayCluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.222Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"f36a45ca-f400-4e46-84bf-9fdebef0a79b","rayCluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.222Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"dcc5e4d1-0c92-4712-af89-1185f5c2b703"}
{"level":"info","ts":"2026-07-13T16:05:34.222Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"dcc5e4d1-0c92-4712-af89-1185f5c2b703","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:05:34.222Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ed541ae0-cf3c-464e-b261-ce6f93a1ad06","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:34.222Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ed541ae0-cf3c-464e-b261-ce6f93a1ad06","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.222Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ed541ae0-cf3c-464e-b261-ce6f93a1ad06","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:34.222Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ed541ae0-cf3c-464e-b261-ce6f93a1ad06"}
{"level":"info","ts":"2026-07-13T16:05:34.222Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ed541ae0-cf3c-464e-b261-ce6f93a1ad06","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:05:34.226Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"4228a014-06b5-45c5-bf8f-9053452063ef","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:34.226Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"4228a014-06b5-45c5-bf8f-9053452063ef","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.226Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"4228a014-06b5-45c5-bf8f-9053452063ef","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:34.226Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"4228a014-06b5-45c5-bf8f-9053452063ef"}
{"level":"info","ts":"2026-07-13T16:05:34.226Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"4228a014-06b5-45c5-bf8f-9053452063ef","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:05:34.229Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"7afe5749-b370-471c-994d-b02435dca24a","name":"stop-b592q-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:05:34.229Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"6be0f9be-35bd-4809-bcf4-e4f00f775518","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.230Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"7876343e-1168-4055-a0fa-c3d0f8b3cbc7","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.230Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"2d93c048-1f94-4f40-ad82-f691c223cb65","name":"stop-b592q","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-b592q\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-x5qvg","name":"stop-b592q"}}
{"level":"info","ts":"2026-07-13T16:05:34.230Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"2d93c048-1f94-4f40-ad82-f691c223cb65"}
{"level":"error","ts":"2026-07-13T16:05:34.230Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"2d93c048-1f94-4f40-ad82-f691c223cb65","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:34.230Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7b5505d0-16e7-42cc-9306-84b91f65847b"}
{"level":"info","ts":"2026-07-13T16:05:34.230Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7b5505d0-16e7-42cc-9306-84b91f65847b"}
{"level":"info","ts":"2026-07-13T16:05:34.230Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7b5505d0-16e7-42cc-9306-84b91f65847b","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:05:34.230Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7b5505d0-16e7-42cc-9306-84b91f65847b","error":"head certificate secret ray-head-secret-stop-b592q not found"}
{"level":"error","ts":"2026-07-13T16:05:34.230Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7b5505d0-16e7-42cc-9306-84b91f65847b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:34.230Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7b5505d0-16e7-42cc-9306-84b91f65847b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found"}
{"level":"info","ts":"2026-07-13T16:05:34.230Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7b5505d0-16e7-42cc-9306-84b91f65847b","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:34.230Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7b5505d0-16e7-42cc-9306-84b91f65847b","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:34.230Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7b5505d0-16e7-42cc-9306-84b91f65847b","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:34.230Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7b5505d0-16e7-42cc-9306-84b91f65847b","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T16:05:34.230Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7b5505d0-16e7-42cc-9306-84b91f65847b","name":"stop-b592q","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:05:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:05:34.231Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"f36a45ca-f400-4e46-84bf-9fdebef0a79b","rayCluster":"stop-b592q","secret":"ca-secret"}
{"level":"info","ts":"2026-07-13T16:05:34.231Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"f36a45ca-f400-4e46-84bf-9fdebef0a79b","rayCluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.238Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"7afe5749-b370-471c-994d-b02435dca24a","name":"kuberay-gateway-access","namespace":"test-ns-x5qvg","operation":"created"}
{"level":"info","ts":"2026-07-13T16:05:34.241Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ea691cad-ba4b-4b39-8538-7998752dfa7c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:34.241Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ea691cad-ba4b-4b39-8538-7998752dfa7c","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.241Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ea691cad-ba4b-4b39-8538-7998752dfa7c","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:34.241Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ea691cad-ba4b-4b39-8538-7998752dfa7c"}
{"level":"info","ts":"2026-07-13T16:05:34.241Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ea691cad-ba4b-4b39-8538-7998752dfa7c","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:05:34.242Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7b5505d0-16e7-42cc-9306-84b91f65847b"}
{"level":"error","ts":"2026-07-13T16:05:34.242Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7b5505d0-16e7-42cc-9306-84b91f65847b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:34.242Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ed6b96af-ac86-4a27-bc0e-3812d24ce00b"}
{"level":"info","ts":"2026-07-13T16:05:34.242Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ed6b96af-ac86-4a27-bc0e-3812d24ce00b"}
{"level":"info","ts":"2026-07-13T16:05:34.242Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ed6b96af-ac86-4a27-bc0e-3812d24ce00b","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:05:34.242Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ed6b96af-ac86-4a27-bc0e-3812d24ce00b","error":"head certificate secret ray-head-secret-stop-b592q not found"}
{"level":"error","ts":"2026-07-13T16:05:34.242Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ed6b96af-ac86-4a27-bc0e-3812d24ce00b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:34.242Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ed6b96af-ac86-4a27-bc0e-3812d24ce00b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found"}
{"level":"info","ts":"2026-07-13T16:05:34.321Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ed6b96af-ac86-4a27-bc0e-3812d24ce00b","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:34.321Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ed6b96af-ac86-4a27-bc0e-3812d24ce00b","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:34.321Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ed6b96af-ac86-4a27-bc0e-3812d24ce00b","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:34.321Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"7876343e-1168-4055-a0fa-c3d0f8b3cbc7","name":"stop-b592q-head"}
{"level":"info","ts":"2026-07-13T16:05:34.321Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ed6b96af-ac86-4a27-bc0e-3812d24ce00b"}
{"level":"error","ts":"2026-07-13T16:05:34.321Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ed6b96af-ac86-4a27-bc0e-3812d24ce00b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:34.321Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"06c017b8-dd40-4380-995b-3058c5d2ea81"}
{"level":"info","ts":"2026-07-13T16:05:34.321Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"06c017b8-dd40-4380-995b-3058c5d2ea81"}
{"level":"info","ts":"2026-07-13T16:05:34.321Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"06c017b8-dd40-4380-995b-3058c5d2ea81","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:05:34.321Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"06c017b8-dd40-4380-995b-3058c5d2ea81","error":"head certificate secret ray-head-secret-stop-b592q not found"}
{"level":"error","ts":"2026-07-13T16:05:34.321Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"06c017b8-dd40-4380-995b-3058c5d2ea81","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:34.321Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"06c017b8-dd40-4380-995b-3058c5d2ea81","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found"}
{"level":"info","ts":"2026-07-13T16:05:34.321Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"f36a45ca-f400-4e46-84bf-9fdebef0a79b","rayCluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.321Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"06c017b8-dd40-4380-995b-3058c5d2ea81","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:34.321Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"06c017b8-dd40-4380-995b-3058c5d2ea81","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:34.321Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"06c017b8-dd40-4380-995b-3058c5d2ea81","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:34.321Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"06c017b8-dd40-4380-995b-3058c5d2ea81"}
{"level":"error","ts":"2026-07-13T16:05:34.321Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"06c017b8-dd40-4380-995b-3058c5d2ea81","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:34.322Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"7afe5749-b370-471c-994d-b02435dca24a","name":"test-ns-x5qvg-stop-b592q","operation":"created"}
{"level":"info","ts":"2026-07-13T16:05:34.421Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4d13f672-778e-48c1-a3c5-3184505984b6"}
{"level":"info","ts":"2026-07-13T16:05:34.421Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4d13f672-778e-48c1-a3c5-3184505984b6"}
{"level":"info","ts":"2026-07-13T16:05:34.421Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"7afe5749-b370-471c-994d-b02435dca24a","name":"kube-rbac-proxy-config-stop-b592q","operation":"created"}
{"level":"info","ts":"2026-07-13T16:05:34.421Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"f36a45ca-f400-4e46-84bf-9fdebef0a79b","rayCluster":"stop-b592q","podIPs":[]}
{"level":"info","ts":"2026-07-13T16:05:34.421Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4d13f672-778e-48c1-a3c5-3184505984b6","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:05:34.421Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4d13f672-778e-48c1-a3c5-3184505984b6","error":"head certificate secret ray-head-secret-stop-b592q not found"}
{"level":"error","ts":"2026-07-13T16:05:34.421Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4d13f672-778e-48c1-a3c5-3184505984b6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:34.421Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4d13f672-778e-48c1-a3c5-3184505984b6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found"}
{"level":"info","ts":"2026-07-13T16:05:34.421Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4d13f672-778e-48c1-a3c5-3184505984b6","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:34.422Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4d13f672-778e-48c1-a3c5-3184505984b6","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:34.422Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4d13f672-778e-48c1-a3c5-3184505984b6","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:34.422Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4d13f672-778e-48c1-a3c5-3184505984b6"}
{"level":"error","ts":"2026-07-13T16:05:34.422Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4d13f672-778e-48c1-a3c5-3184505984b6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:34.425Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"7876343e-1168-4055-a0fa-c3d0f8b3cbc7","name":"stop-b592q-workers"}
{"level":"info","ts":"2026-07-13T16:05:34.425Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"7876343e-1168-4055-a0fa-c3d0f8b3cbc7","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.425Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"718c2f96-3c1f-467e-8492-00c990882453","cluster":"stop-b592q"}
{"level":"error","ts":"2026-07-13T16:05:34.427Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"7afe5749-b370-471c-994d-b02435dca24a","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-b592q\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:34.427Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"7afe5749-b370-471c-994d-b02435dca24a"}
{"level":"error","ts":"2026-07-13T16:05:34.427Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"7afe5749-b370-471c-994d-b02435dca24a","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-b592q\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:34.427Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"7eb64dc2-3f30-4941-ae05-205d52a52627","namespacedName":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:34.427Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"7eb64dc2-3f30-4941-ae05-205d52a52627","mode":"IntegratedOAuth","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.427Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"7eb64dc2-3f30-4941-ae05-205d52a52627","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.434Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"f36a45ca-f400-4e46-84bf-9fdebef0a79b","rayCluster":"stop-b592q","podIPs":[]}
{"level":"info","ts":"2026-07-13T16:05:34.434Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"f36a45ca-f400-4e46-84bf-9fdebef0a79b"}
{"level":"info","ts":"2026-07-13T16:05:34.434Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"7da944f2-b85b-41bb-ad51-8f0f5ea33d0e","rayCluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.434Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"7da944f2-b85b-41bb-ad51-8f0f5ea33d0e"}
{"level":"info","ts":"2026-07-13T16:05:34.438Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"718c2f96-3c1f-467e-8492-00c990882453","name":"stop-b592q-head"}
{"level":"info","ts":"2026-07-13T16:05:34.439Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"7eb64dc2-3f30-4941-ae05-205d52a52627","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.439Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"0de8f9fe-9e0c-4daf-9f1b-f340b7b2d485","namespacedName":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:34.440Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"0de8f9fe-9e0c-4daf-9f1b-f340b7b2d485","mode":"IntegratedOAuth","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.440Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"0de8f9fe-9e0c-4daf-9f1b-f340b7b2d485","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.440Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"10b42225-d031-4559-aa5b-d0aa518df235","rayCluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.440Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"10b42225-d031-4559-aa5b-d0aa518df235"}
{"level":"info","ts":"2026-07-13T16:05:34.440Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"b8ba2092-eb7b-417d-a9bd-878327c7959c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:34.440Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"b8ba2092-eb7b-417d-a9bd-878327c7959c","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.440Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"b8ba2092-eb7b-417d-a9bd-878327c7959c","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:34.440Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"b8ba2092-eb7b-417d-a9bd-878327c7959c"}
{"level":"info","ts":"2026-07-13T16:05:34.440Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"b8ba2092-eb7b-417d-a9bd-878327c7959c","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:05:34.448Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"0de8f9fe-9e0c-4daf-9f1b-f340b7b2d485","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.521Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"5d3af26a-a5ba-4716-898e-25c033e679ed"}
{"level":"info","ts":"2026-07-13T16:05:34.521Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"5d3af26a-a5ba-4716-898e-25c033e679ed"}
{"level":"info","ts":"2026-07-13T16:05:34.521Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"5d3af26a-a5ba-4716-898e-25c033e679ed","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:05:34.521Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"5d3af26a-a5ba-4716-898e-25c033e679ed","error":"head certificate secret ray-head-secret-stop-b592q not found"}
{"level":"error","ts":"2026-07-13T16:05:34.521Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"5d3af26a-a5ba-4716-898e-25c033e679ed","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:34.521Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"5d3af26a-a5ba-4716-898e-25c033e679ed","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found"}
{"level":"info","ts":"2026-07-13T16:05:34.521Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"5d3af26a-a5ba-4716-898e-25c033e679ed","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:34.521Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"5d3af26a-a5ba-4716-898e-25c033e679ed","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:34.521Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"5d3af26a-a5ba-4716-898e-25c033e679ed","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:34.521Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"5d3af26a-a5ba-4716-898e-25c033e679ed"}
{"level":"error","ts":"2026-07-13T16:05:34.521Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"5d3af26a-a5ba-4716-898e-25c033e679ed","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:34.526Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"718c2f96-3c1f-467e-8492-00c990882453","name":"stop-b592q-workers"}
{"level":"info","ts":"2026-07-13T16:05:34.526Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"718c2f96-3c1f-467e-8492-00c990882453","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.526Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"70cb9921-3c5b-4723-9b1f-3e09b6543b1a","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.536Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"70cb9921-3c5b-4723-9b1f-3e09b6543b1a","name":"stop-b592q-head"}
{"level":"info","ts":"2026-07-13T16:05:34.547Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"70cb9921-3c5b-4723-9b1f-3e09b6543b1a","name":"stop-b592q-workers"}
{"level":"info","ts":"2026-07-13T16:05:34.547Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"70cb9921-3c5b-4723-9b1f-3e09b6543b1a","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:34.682Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"59a86e82-7e40-47ee-ac74-8f798780a7ea"}
{"level":"info","ts":"2026-07-13T16:05:34.682Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"59a86e82-7e40-47ee-ac74-8f798780a7ea"}
{"level":"info","ts":"2026-07-13T16:05:34.682Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"59a86e82-7e40-47ee-ac74-8f798780a7ea","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:05:34.682Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"59a86e82-7e40-47ee-ac74-8f798780a7ea","error":"head certificate secret ray-head-secret-stop-b592q not found"}
{"level":"error","ts":"2026-07-13T16:05:34.682Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"59a86e82-7e40-47ee-ac74-8f798780a7ea","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:34.682Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"59a86e82-7e40-47ee-ac74-8f798780a7ea","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found"}
{"level":"info","ts":"2026-07-13T16:05:34.682Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"59a86e82-7e40-47ee-ac74-8f798780a7ea","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:34.682Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"59a86e82-7e40-47ee-ac74-8f798780a7ea","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:34.682Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"59a86e82-7e40-47ee-ac74-8f798780a7ea","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:34.682Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"59a86e82-7e40-47ee-ac74-8f798780a7ea"}
{"level":"error","ts":"2026-07-13T16:05:34.682Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"59a86e82-7e40-47ee-ac74-8f798780a7ea","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:34.946Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"e154a4c6-4b8a-49e6-bb54-6fe944949840","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:05:34.946Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"e154a4c6-4b8a-49e6-bb54-6fe944949840","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T16:05:34.946Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"e154a4c6-4b8a-49e6-bb54-6fe944949840","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T16:05:34.947Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"e154a4c6-4b8a-49e6-bb54-6fe944949840","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T16:05:35.003Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"6dad2c23-66a3-4940-b06f-eaee836c7cdf"}
{"level":"info","ts":"2026-07-13T16:05:35.003Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"6dad2c23-66a3-4940-b06f-eaee836c7cdf"}
{"level":"info","ts":"2026-07-13T16:05:35.003Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"6dad2c23-66a3-4940-b06f-eaee836c7cdf","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:05:35.003Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"6dad2c23-66a3-4940-b06f-eaee836c7cdf","error":"head certificate secret ray-head-secret-stop-b592q not found"}
{"level":"error","ts":"2026-07-13T16:05:35.003Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"6dad2c23-66a3-4940-b06f-eaee836c7cdf","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:35.003Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"6dad2c23-66a3-4940-b06f-eaee836c7cdf","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found"}
{"level":"info","ts":"2026-07-13T16:05:35.003Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"6dad2c23-66a3-4940-b06f-eaee836c7cdf","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:35.003Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"6dad2c23-66a3-4940-b06f-eaee836c7cdf","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:35.003Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"6dad2c23-66a3-4940-b06f-eaee836c7cdf","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"stop-b592q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:05:35.003Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"6dad2c23-66a3-4940-b06f-eaee836c7cdf"}
{"level":"error","ts":"2026-07-13T16:05:35.003Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"6dad2c23-66a3-4940-b06f-eaee836c7cdf","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-b592q not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:35.644Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949"}
{"level":"info","ts":"2026-07-13T16:05:35.644Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949"}
{"level":"info","ts":"2026-07-13T16:05:35.644Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:05:35.644Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949"}
{"level":"info","ts":"2026-07-13T16:05:35.644Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:05:35.644Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949"}
{"level":"info","ts":"2026-07-13T16:05:35.644Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","mode":"IntegratedOAuth","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:35.644Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","cluster":"stop-b592q","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:05:35.644Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","cluster":"stop-b592q","authType":"IntegratedOAuth","serviceAccount":"stop-b592q-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T16:05:35.644Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"stop-b592q","ray.io/group":"headgroup","ray.io/identifier":"stop-b592q-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:05:35.644Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T16:05:35.644Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T16:05:35.644Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T16:05:35.644Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","enabled":true}
{"level":"info","ts":"2026-07-13T16:05:35.657Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","name":"stop-b592q-head-wfqrz"}
{"level":"info","ts":"2026-07-13T16:05:35.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:05:35.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:05:35.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:05:35.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:05:35.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:05:35.657Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949"}
{"level":"info","ts":"2026-07-13T16:05:35.657Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","nodeType":"worker","rayStartParams":{"address":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:05:35.657Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","rayStartCmd":"ray start  --address=stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:05:35.657Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:05:35.657Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","enabled":true}
{"level":"info","ts":"2026-07-13T16:05:35.669Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","name":"stop-b592q-small-group-worker-9hklq"}
{"level":"info","ts":"2026-07-13T16:05:35.669Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","podName":"stop-b592q-head-wfqrz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:05:35.669Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"stop-b592q-head-svc"},"newHead":{"podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"}}
{"level":"info","ts":"2026-07-13T16:05:35.669Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"fc66df03-1f5d-4c06-86d4-3e6693341949","name":"stop-b592q","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:05:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:05:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4027fad1-35a8-4974-96d1-247df1259818"}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4027fad1-35a8-4974-96d1-247df1259818"}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4027fad1-35a8-4974-96d1-247df1259818","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"46d38e3e-616b-4bc1-8496-958490aa1eaa","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"0d45ce50-a036-4743-99cc-8e1ad10f7761","rayCluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4027fad1-35a8-4974-96d1-247df1259818","Found 1 head Pod":"stop-b592q-head-wfqrz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4027fad1-35a8-4974-96d1-247df1259818","head Pod":"stop-b592q-head-wfqrz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-b592q-head-wfqrz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"c3200363-608f-4065-bb08-d9ab9e4fce67","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"c3200363-608f-4065-bb08-d9ab9e4fce67","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"c3200363-608f-4065-bb08-d9ab9e4fce67","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4027fad1-35a8-4974-96d1-247df1259818","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4027fad1-35a8-4974-96d1-247df1259818","worker Pod":"stop-b592q-small-group-worker-9hklq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-b592q-small-group-worker-9hklq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4027fad1-35a8-4974-96d1-247df1259818","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4027fad1-35a8-4974-96d1-247df1259818","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4027fad1-35a8-4974-96d1-247df1259818","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"0d45ce50-a036-4743-99cc-8e1ad10f7761"}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4027fad1-35a8-4974-96d1-247df1259818","podName":"stop-b592q-head-wfqrz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"c3200363-608f-4065-bb08-d9ab9e4fce67"}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"c3200363-608f-4065-bb08-d9ab9e4fce67","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4027fad1-35a8-4974-96d1-247df1259818","podName":"stop-b592q-head-wfqrz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4027fad1-35a8-4974-96d1-247df1259818","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"stop-b592q-head-svc"},"newHead":{"podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"}}
{"level":"info","ts":"2026-07-13T16:05:35.683Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4027fad1-35a8-4974-96d1-247df1259818","name":"stop-b592q","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:05:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:05:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:05:35.721Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"46d38e3e-616b-4bc1-8496-958490aa1eaa","name":"stop-b592q-head"}
{"level":"info","ts":"2026-07-13T16:05:35.721Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4027fad1-35a8-4974-96d1-247df1259818","name":"stop-b592q","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-b592q\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-x5qvg","name":"stop-b592q"}}
{"level":"info","ts":"2026-07-13T16:05:35.721Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4027fad1-35a8-4974-96d1-247df1259818"}
{"level":"error","ts":"2026-07-13T16:05:35.721Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"4027fad1-35a8-4974-96d1-247df1259818","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-b592q\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:35.721Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ca0630e7-faeb-4042-ab90-2cd32bba5197"}
{"level":"info","ts":"2026-07-13T16:05:35.721Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ca0630e7-faeb-4042-ab90-2cd32bba5197"}
{"level":"info","ts":"2026-07-13T16:05:35.721Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ca0630e7-faeb-4042-ab90-2cd32bba5197","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:05:35.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ca0630e7-faeb-4042-ab90-2cd32bba5197","Found 1 head Pod":"stop-b592q-head-wfqrz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:05:35.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ca0630e7-faeb-4042-ab90-2cd32bba5197","head Pod":"stop-b592q-head-wfqrz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-b592q-head-wfqrz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:05:35.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ca0630e7-faeb-4042-ab90-2cd32bba5197","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:05:35.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ca0630e7-faeb-4042-ab90-2cd32bba5197","worker Pod":"stop-b592q-small-group-worker-9hklq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-b592q-small-group-worker-9hklq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:05:35.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ca0630e7-faeb-4042-ab90-2cd32bba5197","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:05:35.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ca0630e7-faeb-4042-ab90-2cd32bba5197","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:05:35.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ca0630e7-faeb-4042-ab90-2cd32bba5197","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:05:35.721Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ca0630e7-faeb-4042-ab90-2cd32bba5197","podName":"stop-b592q-head-wfqrz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:05:35.721Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ca0630e7-faeb-4042-ab90-2cd32bba5197","podName":"stop-b592q-head-wfqrz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:05:35.721Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ca0630e7-faeb-4042-ab90-2cd32bba5197","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T16:05:35.722Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ca0630e7-faeb-4042-ab90-2cd32bba5197","name":"stop-b592q","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:05:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:05:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:05:35.732Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"46d38e3e-616b-4bc1-8496-958490aa1eaa","name":"stop-b592q-workers"}
{"level":"info","ts":"2026-07-13T16:05:35.732Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"46d38e3e-616b-4bc1-8496-958490aa1eaa","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:35.733Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b3d2acf3-2eef-44a8-a10e-3543b0fdfa37"}
{"level":"info","ts":"2026-07-13T16:05:35.733Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b3d2acf3-2eef-44a8-a10e-3543b0fdfa37"}
{"level":"info","ts":"2026-07-13T16:05:35.733Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b3d2acf3-2eef-44a8-a10e-3543b0fdfa37","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:05:35.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b3d2acf3-2eef-44a8-a10e-3543b0fdfa37","Found 1 head Pod":"stop-b592q-head-wfqrz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:05:35.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b3d2acf3-2eef-44a8-a10e-3543b0fdfa37","head Pod":"stop-b592q-head-wfqrz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-b592q-head-wfqrz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:05:35.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b3d2acf3-2eef-44a8-a10e-3543b0fdfa37","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:05:35.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b3d2acf3-2eef-44a8-a10e-3543b0fdfa37","worker Pod":"stop-b592q-small-group-worker-9hklq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-b592q-small-group-worker-9hklq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:05:35.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b3d2acf3-2eef-44a8-a10e-3543b0fdfa37","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:05:35.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b3d2acf3-2eef-44a8-a10e-3543b0fdfa37","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:05:35.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b3d2acf3-2eef-44a8-a10e-3543b0fdfa37","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:05:35.733Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b3d2acf3-2eef-44a8-a10e-3543b0fdfa37","podName":"stop-b592q-head-wfqrz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:05:35.733Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b3d2acf3-2eef-44a8-a10e-3543b0fdfa37","podName":"stop-b592q-head-wfqrz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:05:35.733Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b3d2acf3-2eef-44a8-a10e-3543b0fdfa37","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T16:05:35.733Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"e7ab9418-4bf7-4ce2-8667-01a75fca69f7","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:35.733Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"bc2498c8-f071-4641-9e04-6895cd5e1bc4","rayCluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:35.733Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"4ef3f0a3-99ea-4e1a-a9bf-b33d01420f4b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:35.734Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"4ef3f0a3-99ea-4e1a-a9bf-b33d01420f4b","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:35.734Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"4ef3f0a3-99ea-4e1a-a9bf-b33d01420f4b","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:35.733Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b3d2acf3-2eef-44a8-a10e-3543b0fdfa37","name":"stop-b592q","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:05:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:05:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:05:35.734Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"bc2498c8-f071-4641-9e04-6895cd5e1bc4"}
{"level":"info","ts":"2026-07-13T16:05:35.734Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"4ef3f0a3-99ea-4e1a-a9bf-b33d01420f4b"}
{"level":"info","ts":"2026-07-13T16:05:35.734Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"4ef3f0a3-99ea-4e1a-a9bf-b33d01420f4b","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:05:35.742Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b3d2acf3-2eef-44a8-a10e-3543b0fdfa37","name":"stop-b592q","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-b592q\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-x5qvg","name":"stop-b592q"}}
{"level":"info","ts":"2026-07-13T16:05:35.742Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b3d2acf3-2eef-44a8-a10e-3543b0fdfa37"}
{"level":"error","ts":"2026-07-13T16:05:35.742Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b3d2acf3-2eef-44a8-a10e-3543b0fdfa37","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-b592q\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:05:35.747Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ebdab0e8-4b78-488d-a7da-72d5d3fb4b67"}
{"level":"info","ts":"2026-07-13T16:05:35.747Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ebdab0e8-4b78-488d-a7da-72d5d3fb4b67"}
{"level":"info","ts":"2026-07-13T16:05:35.747Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ebdab0e8-4b78-488d-a7da-72d5d3fb4b67","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:05:35.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ebdab0e8-4b78-488d-a7da-72d5d3fb4b67","Found 1 head Pod":"stop-b592q-head-wfqrz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:05:35.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ebdab0e8-4b78-488d-a7da-72d5d3fb4b67","head Pod":"stop-b592q-head-wfqrz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-b592q-head-wfqrz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:05:35.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ebdab0e8-4b78-488d-a7da-72d5d3fb4b67","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:05:35.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ebdab0e8-4b78-488d-a7da-72d5d3fb4b67","worker Pod":"stop-b592q-small-group-worker-9hklq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-b592q-small-group-worker-9hklq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:05:35.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ebdab0e8-4b78-488d-a7da-72d5d3fb4b67","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:05:35.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ebdab0e8-4b78-488d-a7da-72d5d3fb4b67","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:05:35.748Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ebdab0e8-4b78-488d-a7da-72d5d3fb4b67","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:05:35.748Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ebdab0e8-4b78-488d-a7da-72d5d3fb4b67","podName":"stop-b592q-head-wfqrz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:05:35.748Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ebdab0e8-4b78-488d-a7da-72d5d3fb4b67","podName":"stop-b592q-head-wfqrz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:05:35.748Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"e7ab9418-4bf7-4ce2-8667-01a75fca69f7","name":"stop-b592q-head"}
{"level":"info","ts":"2026-07-13T16:05:35.748Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ebdab0e8-4b78-488d-a7da-72d5d3fb4b67","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:05:35.748Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"ebdab0e8-4b78-488d-a7da-72d5d3fb4b67","seconds":300}
{"level":"info","ts":"2026-07-13T16:05:35.756Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"e7ab9418-4bf7-4ce2-8667-01a75fca69f7","name":"stop-b592q-workers"}
{"level":"info","ts":"2026-07-13T16:05:35.756Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"e7ab9418-4bf7-4ce2-8667-01a75fca69f7","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:36.126Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"edab44ef-c08b-48b0-b351-f732420ff307"}
{"level":"info","ts":"2026-07-13T16:05:36.126Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"edab44ef-c08b-48b0-b351-f732420ff307"}
{"level":"info","ts":"2026-07-13T16:05:36.126Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"edab44ef-c08b-48b0-b351-f732420ff307","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:05:36.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"edab44ef-c08b-48b0-b351-f732420ff307","Found 1 head Pod":"stop-b592q-head-wfqrz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:05:36.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"edab44ef-c08b-48b0-b351-f732420ff307","head Pod":"stop-b592q-head-wfqrz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-b592q-head-wfqrz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:05:36.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"edab44ef-c08b-48b0-b351-f732420ff307","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:05:36.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"edab44ef-c08b-48b0-b351-f732420ff307","worker Pod":"stop-b592q-small-group-worker-9hklq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-b592q-small-group-worker-9hklq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:05:36.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"edab44ef-c08b-48b0-b351-f732420ff307","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:05:36.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"edab44ef-c08b-48b0-b351-f732420ff307","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:05:36.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"edab44ef-c08b-48b0-b351-f732420ff307","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:05:36.126Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"edab44ef-c08b-48b0-b351-f732420ff307","podName":"stop-b592q-small-group-worker-9hklq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:05:36.126Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"edab44ef-c08b-48b0-b351-f732420ff307","podName":"stop-b592q-small-group-worker-9hklq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:05:36.127Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"edab44ef-c08b-48b0-b351-f732420ff307","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:05:36.127Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"edab44ef-c08b-48b0-b351-f732420ff307","seconds":300}
{"level":"info","ts":"2026-07-13T16:05:36.129Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"d5f5a96e-2fc6-4a9a-9f29-60ba77850c50"}
{"level":"info","ts":"2026-07-13T16:05:36.129Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"d5f5a96e-2fc6-4a9a-9f29-60ba77850c50"}
{"level":"info","ts":"2026-07-13T16:05:36.129Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"d5f5a96e-2fc6-4a9a-9f29-60ba77850c50","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:05:36.130Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"d5f5a96e-2fc6-4a9a-9f29-60ba77850c50","Found 1 head Pod":"stop-b592q-head-wfqrz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:05:36.130Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"d5f5a96e-2fc6-4a9a-9f29-60ba77850c50","head Pod":"stop-b592q-head-wfqrz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-b592q-head-wfqrz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:05:36.130Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"d5f5a96e-2fc6-4a9a-9f29-60ba77850c50","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:05:36.130Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"d5f5a96e-2fc6-4a9a-9f29-60ba77850c50","worker Pod":"stop-b592q-small-group-worker-9hklq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-b592q-small-group-worker-9hklq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:05:36.130Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"d5f5a96e-2fc6-4a9a-9f29-60ba77850c50","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:05:36.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"d5f5a96e-2fc6-4a9a-9f29-60ba77850c50","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:05:36.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"d5f5a96e-2fc6-4a9a-9f29-60ba77850c50","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:05:36.221Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"d5f5a96e-2fc6-4a9a-9f29-60ba77850c50","podName":"stop-b592q-head-wfqrz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:05:36.221Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"d5f5a96e-2fc6-4a9a-9f29-60ba77850c50","podName":"stop-b592q-head-wfqrz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:05:36.221Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"d5f5a96e-2fc6-4a9a-9f29-60ba77850c50","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:05:36.221Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"d5f5a96e-2fc6-4a9a-9f29-60ba77850c50","seconds":300}
{"level":"info","ts":"2026-07-13T16:05:36.517Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"34f886c7-ad74-45f3-93cd-1a6039c02726"}
{"level":"info","ts":"2026-07-13T16:05:36.517Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"34f886c7-ad74-45f3-93cd-1a6039c02726"}
{"level":"info","ts":"2026-07-13T16:05:36.517Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"34f886c7-ad74-45f3-93cd-1a6039c02726","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:05:36.517Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"34f886c7-ad74-45f3-93cd-1a6039c02726","Found 1 head Pod":"stop-b592q-head-wfqrz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:05:36.517Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"34f886c7-ad74-45f3-93cd-1a6039c02726","head Pod":"stop-b592q-head-wfqrz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-b592q-head-wfqrz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:05:36.517Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"34f886c7-ad74-45f3-93cd-1a6039c02726","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:05:36.517Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"34f886c7-ad74-45f3-93cd-1a6039c02726","worker Pod":"stop-b592q-small-group-worker-9hklq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-b592q-small-group-worker-9hklq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:05:36.517Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"34f886c7-ad74-45f3-93cd-1a6039c02726","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:05:36.517Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"34f886c7-ad74-45f3-93cd-1a6039c02726","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:05:36.517Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"34f886c7-ad74-45f3-93cd-1a6039c02726","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:05:36.517Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"34f886c7-ad74-45f3-93cd-1a6039c02726","podName":"stop-b592q-small-group-worker-9hklq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:05:36.518Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"34f886c7-ad74-45f3-93cd-1a6039c02726","podName":"stop-b592q-small-group-worker-9hklq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:05:36.518Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"34f886c7-ad74-45f3-93cd-1a6039c02726","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"newHead":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"}}
{"level":"info","ts":"2026-07-13T16:05:36.518Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"34f886c7-ad74-45f3-93cd-1a6039c02726","name":"stop-b592q","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:05:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:05:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:05:36.527Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-x5qvg"},"reconcileID":"27e6a036-8a8a-40ed-bdfb-97267f89b945"}
{"level":"info","ts":"2026-07-13T16:05:36.530Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"413ed99c-728a-40f2-9cee-ac49fa039831","rayCluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:36.530Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"dbc4d448-f216-4aec-9bbe-99eb1a908026","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:36.530Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7e56e8bf-0a6b-47d8-b12c-01b56397008d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:36.530Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7e56e8bf-0a6b-47d8-b12c-01b56397008d","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:36.530Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7e56e8bf-0a6b-47d8-b12c-01b56397008d","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:36.530Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"413ed99c-728a-40f2-9cee-ac49fa039831","pod":"stop-b592q-head-wfqrz","ip":"10.133.0.36"}
{"level":"info","ts":"2026-07-13T16:05:36.530Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7e56e8bf-0a6b-47d8-b12c-01b56397008d"}
{"level":"info","ts":"2026-07-13T16:05:36.530Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7e56e8bf-0a6b-47d8-b12c-01b56397008d","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:05:36.542Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"413ed99c-728a-40f2-9cee-ac49fa039831","certificate":"ray-head-cert-stop-b592q","dnsNames":["stop-b592q-head-svc","localhost","stop-b592q-head-svc.test-ns-x5qvg.svc","stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local"],"ipAddresses":["10.133.0.36","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:05:36.545Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"dbc4d448-f216-4aec-9bbe-99eb1a908026","name":"stop-b592q-head"}
{"level":"info","ts":"2026-07-13T16:05:36.555Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"dbc4d448-f216-4aec-9bbe-99eb1a908026","name":"stop-b592q-workers"}
{"level":"info","ts":"2026-07-13T16:05:36.555Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"dbc4d448-f216-4aec-9bbe-99eb1a908026","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:36.556Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"413ed99c-728a-40f2-9cee-ac49fa039831","certificate":"ray-worker-cert-stop-b592q","dnsNames":["stop-b592q-worker-svc","localhost","stop-b592q-small-group","stop-b592q-worker-svc.test-ns-x5qvg.svc","stop-b592q-worker-svc.test-ns-x5qvg.svc.cluster.local","*.stop-b592q-worker-svc.test-ns-x5qvg.svc","*.stop-b592q-worker-svc.test-ns-x5qvg.svc.cluster.local","*-worker-*.test-ns-x5qvg.svc","*-worker-*.test-ns-x5qvg.svc.cluster.local"],"ipAddresses":["10.133.0.36","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:05:36.556Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"413ed99c-728a-40f2-9cee-ac49fa039831","rayCluster":"stop-b592q","podIPs":["10.133.0.36"]}
{"level":"info","ts":"2026-07-13T16:05:36.857Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"976d19ae-6591-4133-af61-eb23a312785d"}
{"level":"info","ts":"2026-07-13T16:05:36.857Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"976d19ae-6591-4133-af61-eb23a312785d"}
{"level":"info","ts":"2026-07-13T16:05:36.857Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"976d19ae-6591-4133-af61-eb23a312785d","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:05:36.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"976d19ae-6591-4133-af61-eb23a312785d","Found 1 head Pod":"stop-b592q-head-wfqrz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:05:36.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"976d19ae-6591-4133-af61-eb23a312785d","head Pod":"stop-b592q-head-wfqrz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-b592q-head-wfqrz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:05:36.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"976d19ae-6591-4133-af61-eb23a312785d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:05:36.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"976d19ae-6591-4133-af61-eb23a312785d","worker Pod":"stop-b592q-small-group-worker-9hklq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-b592q-small-group-worker-9hklq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:05:36.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"976d19ae-6591-4133-af61-eb23a312785d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:05:36.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"976d19ae-6591-4133-af61-eb23a312785d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:05:36.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"976d19ae-6591-4133-af61-eb23a312785d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:05:36.857Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"976d19ae-6591-4133-af61-eb23a312785d","podName":"stop-b592q-head-wfqrz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:05:36.857Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"976d19ae-6591-4133-af61-eb23a312785d","podName":"stop-b592q-head-wfqrz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:05:36.857Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"976d19ae-6591-4133-af61-eb23a312785d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:05:36.857Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"976d19ae-6591-4133-af61-eb23a312785d","seconds":300}
{"level":"info","ts":"2026-07-13T16:05:37.228Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"55b51799-54d2-4acb-a549-308d13bdbb56","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:37.228Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"55b51799-54d2-4acb-a549-308d13bdbb56","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:37.228Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"55b51799-54d2-4acb-a549-308d13bdbb56","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:37.228Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"55b51799-54d2-4acb-a549-308d13bdbb56"}
{"level":"info","ts":"2026-07-13T16:05:37.228Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"55b51799-54d2-4acb-a549-308d13bdbb56","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:05:38.530Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"8f1583d0-1646-4561-a505-1c930955457e"}
{"level":"info","ts":"2026-07-13T16:05:38.530Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"8f1583d0-1646-4561-a505-1c930955457e"}
{"level":"info","ts":"2026-07-13T16:05:38.530Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"8f1583d0-1646-4561-a505-1c930955457e","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:05:38.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"8f1583d0-1646-4561-a505-1c930955457e","Found 1 head Pod":"stop-b592q-head-wfqrz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:05:38.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"8f1583d0-1646-4561-a505-1c930955457e","head Pod":"stop-b592q-head-wfqrz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-b592q-head-wfqrz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:05:38.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"8f1583d0-1646-4561-a505-1c930955457e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:05:38.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"8f1583d0-1646-4561-a505-1c930955457e","worker Pod":"stop-b592q-small-group-worker-9hklq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-b592q-small-group-worker-9hklq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:05:38.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"8f1583d0-1646-4561-a505-1c930955457e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:05:38.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"8f1583d0-1646-4561-a505-1c930955457e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:05:38.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"8f1583d0-1646-4561-a505-1c930955457e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:05:38.530Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"8f1583d0-1646-4561-a505-1c930955457e","podName":"stop-b592q-head-wfqrz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:05:38.531Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"8f1583d0-1646-4561-a505-1c930955457e","podName":"stop-b592q-head-wfqrz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:05:38.531Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"8f1583d0-1646-4561-a505-1c930955457e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:05:38.531Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"8f1583d0-1646-4561-a505-1c930955457e","seconds":300}
{"level":"info","ts":"2026-07-13T16:05:40.229Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"250e451c-7d55-4e16-aeb4-d7ddcf8d2fd3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:40.229Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"250e451c-7d55-4e16-aeb4-d7ddcf8d2fd3","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:40.229Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"250e451c-7d55-4e16-aeb4-d7ddcf8d2fd3","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:40.229Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"250e451c-7d55-4e16-aeb4-d7ddcf8d2fd3"}
{"level":"info","ts":"2026-07-13T16:05:40.229Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"250e451c-7d55-4e16-aeb4-d7ddcf8d2fd3","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:05:43.230Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2e2e6427-5c5f-472d-98c5-9d7303f0da4e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:43.230Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2e2e6427-5c5f-472d-98c5-9d7303f0da4e","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:43.230Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2e2e6427-5c5f-472d-98c5-9d7303f0da4e","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:43.230Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2e2e6427-5c5f-472d-98c5-9d7303f0da4e"}
{"level":"info","ts":"2026-07-13T16:05:43.230Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2e2e6427-5c5f-472d-98c5-9d7303f0da4e","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:05:46.230Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7afc44a9-1dd2-4045-8af3-d91a019f5f26","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:46.230Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7afc44a9-1dd2-4045-8af3-d91a019f5f26","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:46.230Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7afc44a9-1dd2-4045-8af3-d91a019f5f26","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:46.231Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7afc44a9-1dd2-4045-8af3-d91a019f5f26"}
{"level":"info","ts":"2026-07-13T16:05:46.231Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7afc44a9-1dd2-4045-8af3-d91a019f5f26","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:05:49.231Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d802cac4-9aff-44e5-a6db-ab978126f888","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:49.232Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d802cac4-9aff-44e5-a6db-ab978126f888","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:49.232Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d802cac4-9aff-44e5-a6db-ab978126f888","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:49.232Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d802cac4-9aff-44e5-a6db-ab978126f888"}
{"level":"info","ts":"2026-07-13T16:05:49.232Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d802cac4-9aff-44e5-a6db-ab978126f888","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:05:52.232Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2942c86c-9d05-4737-b49f-52ab68b5eaf8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:52.232Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2942c86c-9d05-4737-b49f-52ab68b5eaf8","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:52.232Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2942c86c-9d05-4737-b49f-52ab68b5eaf8","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:52.232Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2942c86c-9d05-4737-b49f-52ab68b5eaf8"}
{"level":"info","ts":"2026-07-13T16:05:52.232Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2942c86c-9d05-4737-b49f-52ab68b5eaf8","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:05:55.233Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a2fe95d0-065a-4c68-b575-acc25fe472bf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:55.233Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a2fe95d0-065a-4c68-b575-acc25fe472bf","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:55.233Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a2fe95d0-065a-4c68-b575-acc25fe472bf","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:55.234Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a2fe95d0-065a-4c68-b575-acc25fe472bf"}
{"level":"info","ts":"2026-07-13T16:05:55.234Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a2fe95d0-065a-4c68-b575-acc25fe472bf","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:05:58.234Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"534055d9-fc3d-4b2f-814a-fc7ef07a3d43","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:05:58.234Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"534055d9-fc3d-4b2f-814a-fc7ef07a3d43","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:05:58.234Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"534055d9-fc3d-4b2f-814a-fc7ef07a3d43","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:05:58.235Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"534055d9-fc3d-4b2f-814a-fc7ef07a3d43"}
{"level":"info","ts":"2026-07-13T16:05:58.235Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"534055d9-fc3d-4b2f-814a-fc7ef07a3d43","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:01.235Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"897c9461-fd2d-4bcf-93f4-6e0bffeff13b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:01.235Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"897c9461-fd2d-4bcf-93f4-6e0bffeff13b","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:01.235Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"897c9461-fd2d-4bcf-93f4-6e0bffeff13b","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:01.235Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"897c9461-fd2d-4bcf-93f4-6e0bffeff13b"}
{"level":"info","ts":"2026-07-13T16:06:01.235Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"897c9461-fd2d-4bcf-93f4-6e0bffeff13b","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:04.236Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5bb13624-f8c6-4fec-ba9c-02e8dcb6eaa3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:04.236Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5bb13624-f8c6-4fec-ba9c-02e8dcb6eaa3","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:04.236Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5bb13624-f8c6-4fec-ba9c-02e8dcb6eaa3","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:04.236Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5bb13624-f8c6-4fec-ba9c-02e8dcb6eaa3"}
{"level":"info","ts":"2026-07-13T16:06:04.236Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5bb13624-f8c6-4fec-ba9c-02e8dcb6eaa3","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:04.434Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"01f6e24b-b7b3-4f83-9446-c14bb93e694d","rayCluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:04.434Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"01f6e24b-b7b3-4f83-9446-c14bb93e694d","pod":"stop-b592q-head-wfqrz","ip":"10.133.0.36"}
{"level":"info","ts":"2026-07-13T16:06:04.434Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"01f6e24b-b7b3-4f83-9446-c14bb93e694d","pod":"stop-b592q-small-group-worker-9hklq","ip":"10.134.0.34"}
{"level":"info","ts":"2026-07-13T16:06:04.449Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"01f6e24b-b7b3-4f83-9446-c14bb93e694d","certificate":"ray-head-cert-stop-b592q","dnsNames":["stop-b592q-head-svc","localhost","stop-b592q-head-svc.test-ns-x5qvg.svc","stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local"],"ipAddresses":["10.133.0.36","10.134.0.34","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:06:04.467Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"01f6e24b-b7b3-4f83-9446-c14bb93e694d","certificate":"ray-worker-cert-stop-b592q","dnsNames":["stop-b592q-worker-svc","localhost","stop-b592q-small-group","stop-b592q-worker-svc.test-ns-x5qvg.svc","stop-b592q-worker-svc.test-ns-x5qvg.svc.cluster.local","*.stop-b592q-worker-svc.test-ns-x5qvg.svc","*.stop-b592q-worker-svc.test-ns-x5qvg.svc.cluster.local","*-worker-*.test-ns-x5qvg.svc","*-worker-*.test-ns-x5qvg.svc.cluster.local"],"ipAddresses":["10.133.0.36","10.134.0.34","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:06:04.467Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"01f6e24b-b7b3-4f83-9446-c14bb93e694d"}
{"level":"info","ts":"2026-07-13T16:06:07.236Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"f2d361f9-dab3-4769-b8dc-dd2d6499cff4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:07.236Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"f2d361f9-dab3-4769-b8dc-dd2d6499cff4","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:07.236Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"f2d361f9-dab3-4769-b8dc-dd2d6499cff4","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:07.237Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"f2d361f9-dab3-4769-b8dc-dd2d6499cff4"}
{"level":"info","ts":"2026-07-13T16:06:07.237Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"f2d361f9-dab3-4769-b8dc-dd2d6499cff4","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:10.237Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"402b7f56-d89b-43ec-b107-36c509a08b56","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:10.237Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"402b7f56-d89b-43ec-b107-36c509a08b56","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:10.237Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"402b7f56-d89b-43ec-b107-36c509a08b56","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:10.237Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"402b7f56-d89b-43ec-b107-36c509a08b56"}
{"level":"info","ts":"2026-07-13T16:06:10.237Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"402b7f56-d89b-43ec-b107-36c509a08b56","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:13.238Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5ef72ce3-cd7b-4d71-84c3-bb80e8f73e44","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:13.238Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5ef72ce3-cd7b-4d71-84c3-bb80e8f73e44","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:13.238Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5ef72ce3-cd7b-4d71-84c3-bb80e8f73e44","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:13.238Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5ef72ce3-cd7b-4d71-84c3-bb80e8f73e44"}
{"level":"info","ts":"2026-07-13T16:06:13.238Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5ef72ce3-cd7b-4d71-84c3-bb80e8f73e44","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:16.238Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2618cc63-a7ab-4b11-816d-ed41a5a00833","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:16.238Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2618cc63-a7ab-4b11-816d-ed41a5a00833","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:16.238Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2618cc63-a7ab-4b11-816d-ed41a5a00833","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:16.238Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2618cc63-a7ab-4b11-816d-ed41a5a00833"}
{"level":"info","ts":"2026-07-13T16:06:16.238Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2618cc63-a7ab-4b11-816d-ed41a5a00833","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:19.239Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"b1dc7c91-14a1-4d2d-9ca0-6f2d7051ab1e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:19.239Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"b1dc7c91-14a1-4d2d-9ca0-6f2d7051ab1e","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:19.239Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"b1dc7c91-14a1-4d2d-9ca0-6f2d7051ab1e","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:19.239Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"b1dc7c91-14a1-4d2d-9ca0-6f2d7051ab1e"}
{"level":"info","ts":"2026-07-13T16:06:19.239Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"b1dc7c91-14a1-4d2d-9ca0-6f2d7051ab1e","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:22.240Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"e3f946bf-88eb-429f-a8d0-e30be87f9481","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:22.240Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"e3f946bf-88eb-429f-a8d0-e30be87f9481","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:22.240Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"e3f946bf-88eb-429f-a8d0-e30be87f9481","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:22.240Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"e3f946bf-88eb-429f-a8d0-e30be87f9481"}
{"level":"info","ts":"2026-07-13T16:06:22.240Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"e3f946bf-88eb-429f-a8d0-e30be87f9481","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:25.240Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d90b1d6e-a9e8-47fa-9ef5-9a7c95ef769a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:25.241Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d90b1d6e-a9e8-47fa-9ef5-9a7c95ef769a","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:25.241Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d90b1d6e-a9e8-47fa-9ef5-9a7c95ef769a","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:25.241Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d90b1d6e-a9e8-47fa-9ef5-9a7c95ef769a"}
{"level":"info","ts":"2026-07-13T16:06:25.241Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d90b1d6e-a9e8-47fa-9ef5-9a7c95ef769a","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:28.242Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7b4d724d-1998-40d5-b163-2c4cac559a0d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:28.242Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7b4d724d-1998-40d5-b163-2c4cac559a0d","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:28.242Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7b4d724d-1998-40d5-b163-2c4cac559a0d","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:28.242Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7b4d724d-1998-40d5-b163-2c4cac559a0d"}
{"level":"info","ts":"2026-07-13T16:06:28.242Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7b4d724d-1998-40d5-b163-2c4cac559a0d","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:31.242Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"3d0ab6c3-502d-4827-929d-5b1a8d317ee5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:31.242Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"3d0ab6c3-502d-4827-929d-5b1a8d317ee5","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:31.242Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"3d0ab6c3-502d-4827-929d-5b1a8d317ee5","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:31.242Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"3d0ab6c3-502d-4827-929d-5b1a8d317ee5"}
{"level":"info","ts":"2026-07-13T16:06:31.242Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"3d0ab6c3-502d-4827-929d-5b1a8d317ee5","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:34.243Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a404af15-a462-454c-8514-98c5ae157c70","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:34.243Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a404af15-a462-454c-8514-98c5ae157c70","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:34.243Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a404af15-a462-454c-8514-98c5ae157c70","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:34.243Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a404af15-a462-454c-8514-98c5ae157c70"}
{"level":"info","ts":"2026-07-13T16:06:34.243Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a404af15-a462-454c-8514-98c5ae157c70","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:34.467Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"d6d92208-6ddf-427f-ab74-f655efe61487","rayCluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:34.468Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"d6d92208-6ddf-427f-ab74-f655efe61487","pod":"stop-b592q-head-wfqrz","ip":"10.133.0.36"}
{"level":"info","ts":"2026-07-13T16:06:34.468Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"d6d92208-6ddf-427f-ab74-f655efe61487","pod":"stop-b592q-small-group-worker-9hklq","ip":"10.134.0.34"}
{"level":"info","ts":"2026-07-13T16:06:34.468Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"d6d92208-6ddf-427f-ab74-f655efe61487","rayCluster":"stop-b592q","podIPs":["10.133.0.36","10.134.0.34"]}
{"level":"info","ts":"2026-07-13T16:06:34.947Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"d1da90e2-1d4e-47e9-8ec0-dee05bac1f50","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:06:34.947Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"d1da90e2-1d4e-47e9-8ec0-dee05bac1f50","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T16:06:34.947Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"d1da90e2-1d4e-47e9-8ec0-dee05bac1f50","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T16:06:34.947Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"d1da90e2-1d4e-47e9-8ec0-dee05bac1f50","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T16:06:37.244Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6ee7d41e-a8ce-4690-a31d-6bb809cc2ecc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:37.244Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6ee7d41e-a8ce-4690-a31d-6bb809cc2ecc","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:37.244Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6ee7d41e-a8ce-4690-a31d-6bb809cc2ecc","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:37.244Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6ee7d41e-a8ce-4690-a31d-6bb809cc2ecc"}
{"level":"info","ts":"2026-07-13T16:06:37.244Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6ee7d41e-a8ce-4690-a31d-6bb809cc2ecc","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:40.245Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"3912cb46-d575-4005-9085-36e0b6fc16f4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:40.245Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"3912cb46-d575-4005-9085-36e0b6fc16f4","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:40.245Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"3912cb46-d575-4005-9085-36e0b6fc16f4","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:40.245Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"3912cb46-d575-4005-9085-36e0b6fc16f4"}
{"level":"info","ts":"2026-07-13T16:06:40.245Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"3912cb46-d575-4005-9085-36e0b6fc16f4","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:43.245Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"11ae6464-f3cf-4f3b-b790-214778e7f958","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:43.245Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"11ae6464-f3cf-4f3b-b790-214778e7f958","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:43.245Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"11ae6464-f3cf-4f3b-b790-214778e7f958","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:43.246Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"11ae6464-f3cf-4f3b-b790-214778e7f958"}
{"level":"info","ts":"2026-07-13T16:06:43.246Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"11ae6464-f3cf-4f3b-b790-214778e7f958","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:46.246Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"43e6a480-f937-4d80-9c11-70820290d377","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:46.246Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"43e6a480-f937-4d80-9c11-70820290d377","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:46.246Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"43e6a480-f937-4d80-9c11-70820290d377","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:46.246Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"43e6a480-f937-4d80-9c11-70820290d377"}
{"level":"info","ts":"2026-07-13T16:06:46.247Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"43e6a480-f937-4d80-9c11-70820290d377","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:46.767Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"be1bc787-557b-4045-9c80-488f4a82d976"}
{"level":"info","ts":"2026-07-13T16:06:46.767Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"be1bc787-557b-4045-9c80-488f4a82d976"}
{"level":"info","ts":"2026-07-13T16:06:46.767Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"be1bc787-557b-4045-9c80-488f4a82d976","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:06:46.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"be1bc787-557b-4045-9c80-488f4a82d976","Found 1 head Pod":"stop-b592q-head-wfqrz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:06:46.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"be1bc787-557b-4045-9c80-488f4a82d976","head Pod":"stop-b592q-head-wfqrz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-b592q-head-wfqrz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:06:46.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"be1bc787-557b-4045-9c80-488f4a82d976","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:06:46.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"be1bc787-557b-4045-9c80-488f4a82d976","worker Pod":"stop-b592q-small-group-worker-9hklq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-b592q-small-group-worker-9hklq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:06:46.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"be1bc787-557b-4045-9c80-488f4a82d976","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:06:46.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"be1bc787-557b-4045-9c80-488f4a82d976","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:06:46.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"be1bc787-557b-4045-9c80-488f4a82d976","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:06:46.767Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"be1bc787-557b-4045-9c80-488f4a82d976","podName":"stop-b592q-head-wfqrz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:06:46.767Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"be1bc787-557b-4045-9c80-488f4a82d976","podName":"stop-b592q-head-wfqrz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:06:46.767Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"be1bc787-557b-4045-9c80-488f4a82d976","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:06:46.767Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"be1bc787-557b-4045-9c80-488f4a82d976","seconds":300}
{"level":"info","ts":"2026-07-13T16:06:47.775Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"0a12266e-657a-433e-9b3d-155873803e4c"}
{"level":"info","ts":"2026-07-13T16:06:47.775Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"0a12266e-657a-433e-9b3d-155873803e4c"}
{"level":"info","ts":"2026-07-13T16:06:47.775Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"0a12266e-657a-433e-9b3d-155873803e4c","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:06:47.775Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"0a12266e-657a-433e-9b3d-155873803e4c","Found 1 head Pod":"stop-b592q-head-wfqrz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:06:47.775Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"0a12266e-657a-433e-9b3d-155873803e4c","head Pod":"stop-b592q-head-wfqrz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-b592q-head-wfqrz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:06:47.775Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"0a12266e-657a-433e-9b3d-155873803e4c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:06:47.775Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"0a12266e-657a-433e-9b3d-155873803e4c","worker Pod":"stop-b592q-small-group-worker-9hklq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-b592q-small-group-worker-9hklq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:06:47.775Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"0a12266e-657a-433e-9b3d-155873803e4c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:06:47.775Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"0a12266e-657a-433e-9b3d-155873803e4c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:06:47.775Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"0a12266e-657a-433e-9b3d-155873803e4c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:06:47.775Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"0a12266e-657a-433e-9b3d-155873803e4c","podName":"stop-b592q-head-wfqrz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:05:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:06:47.775Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"0a12266e-657a-433e-9b3d-155873803e4c","podName":"stop-b592q-head-wfqrz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:05:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:06:47.775Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"0a12266e-657a-433e-9b3d-155873803e4c","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T16:06:47.775Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"0a12266e-657a-433e-9b3d-155873803e4c","name":"stop-b592q","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:05:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:06:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:06:47.789Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"72d520a0-f59a-403d-bfb6-b3eaaec017cf","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:47.789Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"5582f7d0-cbb9-46b6-bfd9-f3fe201e648c","rayCluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:47.789Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"94faf634-3188-4f63-a91d-2175ba14a152","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:47.789Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"94faf634-3188-4f63-a91d-2175ba14a152","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:47.789Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"94faf634-3188-4f63-a91d-2175ba14a152","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:47.789Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"5582f7d0-cbb9-46b6-bfd9-f3fe201e648c","pod":"stop-b592q-head-wfqrz","ip":"10.133.0.36"}
{"level":"info","ts":"2026-07-13T16:06:47.790Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"5582f7d0-cbb9-46b6-bfd9-f3fe201e648c","pod":"stop-b592q-small-group-worker-9hklq","ip":"10.134.0.34"}
{"level":"info","ts":"2026-07-13T16:06:47.790Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"5582f7d0-cbb9-46b6-bfd9-f3fe201e648c","rayCluster":"stop-b592q","podIPs":["10.133.0.36","10.134.0.34"]}
{"level":"info","ts":"2026-07-13T16:06:47.790Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"94faf634-3188-4f63-a91d-2175ba14a152"}
{"level":"info","ts":"2026-07-13T16:06:47.790Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"94faf634-3188-4f63-a91d-2175ba14a152","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:47.804Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"72d520a0-f59a-403d-bfb6-b3eaaec017cf","name":"stop-b592q-head"}
{"level":"info","ts":"2026-07-13T16:06:47.813Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"72d520a0-f59a-403d-bfb6-b3eaaec017cf","name":"stop-b592q-workers"}
{"level":"info","ts":"2026-07-13T16:06:47.813Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"72d520a0-f59a-403d-bfb6-b3eaaec017cf","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:49.247Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"84d5605f-8927-43f4-8672-8d5531cd9099","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:49.247Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"84d5605f-8927-43f4-8672-8d5531cd9099","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:49.247Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"84d5605f-8927-43f4-8672-8d5531cd9099","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:49.248Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"84d5605f-8927-43f4-8672-8d5531cd9099"}
{"level":"info","ts":"2026-07-13T16:06:49.248Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"84d5605f-8927-43f4-8672-8d5531cd9099","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:49.790Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"cbd8fb4e-75a7-4325-95e3-67175eda94f4"}
{"level":"info","ts":"2026-07-13T16:06:49.790Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"cbd8fb4e-75a7-4325-95e3-67175eda94f4"}
{"level":"info","ts":"2026-07-13T16:06:49.790Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"cbd8fb4e-75a7-4325-95e3-67175eda94f4","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:06:49.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"cbd8fb4e-75a7-4325-95e3-67175eda94f4","Found 1 head Pod":"stop-b592q-head-wfqrz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:06:49.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"cbd8fb4e-75a7-4325-95e3-67175eda94f4","head Pod":"stop-b592q-head-wfqrz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-b592q-head-wfqrz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:06:49.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"cbd8fb4e-75a7-4325-95e3-67175eda94f4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:06:49.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"cbd8fb4e-75a7-4325-95e3-67175eda94f4","worker Pod":"stop-b592q-small-group-worker-9hklq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-b592q-small-group-worker-9hklq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:06:49.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"cbd8fb4e-75a7-4325-95e3-67175eda94f4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:06:49.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"cbd8fb4e-75a7-4325-95e3-67175eda94f4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:06:49.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"cbd8fb4e-75a7-4325-95e3-67175eda94f4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:06:49.790Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"cbd8fb4e-75a7-4325-95e3-67175eda94f4","podName":"stop-b592q-small-group-worker-9hklq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:06:49.790Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"cbd8fb4e-75a7-4325-95e3-67175eda94f4","podName":"stop-b592q-small-group-worker-9hklq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:06:49.790Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"cbd8fb4e-75a7-4325-95e3-67175eda94f4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:06:49.790Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"cbd8fb4e-75a7-4325-95e3-67175eda94f4","seconds":300}
{"level":"info","ts":"2026-07-13T16:06:52.248Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"c63b4684-f2a5-4d79-9227-b1aae6f49897","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:52.248Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"c63b4684-f2a5-4d79-9227-b1aae6f49897","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:52.249Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"c63b4684-f2a5-4d79-9227-b1aae6f49897","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:52.249Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"c63b4684-f2a5-4d79-9227-b1aae6f49897"}
{"level":"info","ts":"2026-07-13T16:06:52.249Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"c63b4684-f2a5-4d79-9227-b1aae6f49897","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:55.250Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"c418e915-1dca-40ae-9b09-168e92ef94f3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:55.250Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"c418e915-1dca-40ae-9b09-168e92ef94f3","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:55.250Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"c418e915-1dca-40ae-9b09-168e92ef94f3","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:55.250Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"c418e915-1dca-40ae-9b09-168e92ef94f3"}
{"level":"info","ts":"2026-07-13T16:06:55.250Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"c418e915-1dca-40ae-9b09-168e92ef94f3","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:58.250Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7e12c403-f88b-42af-8254-52eab349e22d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:58.250Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7e12c403-f88b-42af-8254-52eab349e22d","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:58.250Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7e12c403-f88b-42af-8254-52eab349e22d","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:58.250Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7e12c403-f88b-42af-8254-52eab349e22d"}
{"level":"info","ts":"2026-07-13T16:06:58.250Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7e12c403-f88b-42af-8254-52eab349e22d","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:58.803Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"dec7bb94-d21d-48dd-a0b2-10766833622f"}
{"level":"info","ts":"2026-07-13T16:06:58.803Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"dec7bb94-d21d-48dd-a0b2-10766833622f"}
{"level":"info","ts":"2026-07-13T16:06:58.803Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"dec7bb94-d21d-48dd-a0b2-10766833622f","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:06:58.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"dec7bb94-d21d-48dd-a0b2-10766833622f","Found 1 head Pod":"stop-b592q-head-wfqrz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:06:58.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"dec7bb94-d21d-48dd-a0b2-10766833622f","head Pod":"stop-b592q-head-wfqrz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-b592q-head-wfqrz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:06:58.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"dec7bb94-d21d-48dd-a0b2-10766833622f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:06:58.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"dec7bb94-d21d-48dd-a0b2-10766833622f","worker Pod":"stop-b592q-small-group-worker-9hklq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-b592q-small-group-worker-9hklq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:06:58.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"dec7bb94-d21d-48dd-a0b2-10766833622f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:06:58.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"dec7bb94-d21d-48dd-a0b2-10766833622f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:06:58.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"dec7bb94-d21d-48dd-a0b2-10766833622f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:06:58.803Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"dec7bb94-d21d-48dd-a0b2-10766833622f","podName":"stop-b592q-small-group-worker-9hklq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:06:58.803Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"dec7bb94-d21d-48dd-a0b2-10766833622f","podName":"stop-b592q-small-group-worker-9hklq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:06:58.803Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"dec7bb94-d21d-48dd-a0b2-10766833622f","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T16:06:58.803Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"dec7bb94-d21d-48dd-a0b2-10766833622f","name":"stop-b592q","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:06:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:06:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:06:58.814Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"c2d51322-7826-45bd-88c9-fabc35fab886","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:58.814Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"62b3be3f-6f3f-4a8b-be44-0a7b8c823830","rayCluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:58.814Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a98de5c7-2086-47fb-aafe-ca84c6fafff2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:06:58.814Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a98de5c7-2086-47fb-aafe-ca84c6fafff2","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:06:58.814Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a98de5c7-2086-47fb-aafe-ca84c6fafff2","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:06:58.814Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"62b3be3f-6f3f-4a8b-be44-0a7b8c823830","pod":"stop-b592q-head-wfqrz","ip":"10.133.0.36"}
{"level":"info","ts":"2026-07-13T16:06:58.814Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"62b3be3f-6f3f-4a8b-be44-0a7b8c823830","pod":"stop-b592q-small-group-worker-9hklq","ip":"10.134.0.34"}
{"level":"info","ts":"2026-07-13T16:06:58.815Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"62b3be3f-6f3f-4a8b-be44-0a7b8c823830","rayCluster":"stop-b592q","podIPs":["10.133.0.36","10.134.0.34"]}
{"level":"info","ts":"2026-07-13T16:06:58.815Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a98de5c7-2086-47fb-aafe-ca84c6fafff2"}
{"level":"info","ts":"2026-07-13T16:06:58.815Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a98de5c7-2086-47fb-aafe-ca84c6fafff2","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:06:58.823Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"c2d51322-7826-45bd-88c9-fabc35fab886","name":"stop-b592q-head"}
{"level":"info","ts":"2026-07-13T16:06:58.834Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"c2d51322-7826-45bd-88c9-fabc35fab886","name":"stop-b592q-workers"}
{"level":"info","ts":"2026-07-13T16:06:58.834Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"c2d51322-7826-45bd-88c9-fabc35fab886","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:00.815Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"e3b13d34-3b5c-4cbf-ad93-4ef0c0320926"}
{"level":"info","ts":"2026-07-13T16:07:00.815Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"e3b13d34-3b5c-4cbf-ad93-4ef0c0320926"}
{"level":"info","ts":"2026-07-13T16:07:00.815Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"e3b13d34-3b5c-4cbf-ad93-4ef0c0320926","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:07:00.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"e3b13d34-3b5c-4cbf-ad93-4ef0c0320926","Found 1 head Pod":"stop-b592q-head-wfqrz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:07:00.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"e3b13d34-3b5c-4cbf-ad93-4ef0c0320926","head Pod":"stop-b592q-head-wfqrz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-b592q-head-wfqrz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:07:00.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"e3b13d34-3b5c-4cbf-ad93-4ef0c0320926","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:07:00.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"e3b13d34-3b5c-4cbf-ad93-4ef0c0320926","worker Pod":"stop-b592q-small-group-worker-9hklq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-b592q-small-group-worker-9hklq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:07:00.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"e3b13d34-3b5c-4cbf-ad93-4ef0c0320926","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:07:00.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"e3b13d34-3b5c-4cbf-ad93-4ef0c0320926","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:07:00.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"e3b13d34-3b5c-4cbf-ad93-4ef0c0320926","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:07:00.815Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"e3b13d34-3b5c-4cbf-ad93-4ef0c0320926","podName":"stop-b592q-small-group-worker-9hklq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:07:00.815Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"e3b13d34-3b5c-4cbf-ad93-4ef0c0320926","podName":"stop-b592q-small-group-worker-9hklq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:07:00.815Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"e3b13d34-3b5c-4cbf-ad93-4ef0c0320926","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:07:00.815Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"e3b13d34-3b5c-4cbf-ad93-4ef0c0320926","seconds":300}
{"level":"info","ts":"2026-07-13T16:07:01.251Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"f417626b-c43f-46b6-afdf-2a3164303c9f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:01.251Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"f417626b-c43f-46b6-afdf-2a3164303c9f","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:01.251Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"f417626b-c43f-46b6-afdf-2a3164303c9f","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:01.251Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"f417626b-c43f-46b6-afdf-2a3164303c9f"}
{"level":"info","ts":"2026-07-13T16:07:01.251Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"f417626b-c43f-46b6-afdf-2a3164303c9f","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:07:04.252Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"77fb803c-9b57-42a3-9549-9de39bd70fb8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:04.252Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"77fb803c-9b57-42a3-9549-9de39bd70fb8","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:04.252Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"77fb803c-9b57-42a3-9549-9de39bd70fb8","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:04.252Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"77fb803c-9b57-42a3-9549-9de39bd70fb8"}
{"level":"info","ts":"2026-07-13T16:07:04.252Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"77fb803c-9b57-42a3-9549-9de39bd70fb8","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:07:07.216Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7962a01b-4fe3-4f14-834a-39e452476029"}
{"level":"info","ts":"2026-07-13T16:07:07.216Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7962a01b-4fe3-4f14-834a-39e452476029"}
{"level":"info","ts":"2026-07-13T16:07:07.216Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7962a01b-4fe3-4f14-834a-39e452476029","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:07:07.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7962a01b-4fe3-4f14-834a-39e452476029","Found 1 head Pod":"stop-b592q-head-wfqrz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:07:07.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7962a01b-4fe3-4f14-834a-39e452476029","head Pod":"stop-b592q-head-wfqrz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-b592q-head-wfqrz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:07:07.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7962a01b-4fe3-4f14-834a-39e452476029","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:07:07.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7962a01b-4fe3-4f14-834a-39e452476029","worker Pod":"stop-b592q-small-group-worker-9hklq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-b592q-small-group-worker-9hklq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:07:07.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7962a01b-4fe3-4f14-834a-39e452476029","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:07:07.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7962a01b-4fe3-4f14-834a-39e452476029","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:07:07.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7962a01b-4fe3-4f14-834a-39e452476029","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:07:07.216Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7962a01b-4fe3-4f14-834a-39e452476029","podName":"stop-b592q-small-group-worker-9hklq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:07:07.216Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7962a01b-4fe3-4f14-834a-39e452476029","podName":"stop-b592q-small-group-worker-9hklq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:07:07.216Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7962a01b-4fe3-4f14-834a-39e452476029","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:07:07.216Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"7962a01b-4fe3-4f14-834a-39e452476029","seconds":300}
{"level":"info","ts":"2026-07-13T16:07:07.253Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"0ce47ee6-1ce0-4de2-b059-4e4eb9ba0f37","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:07.253Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"0ce47ee6-1ce0-4de2-b059-4e4eb9ba0f37","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:07.253Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"0ce47ee6-1ce0-4de2-b059-4e4eb9ba0f37","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:07.253Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"0ce47ee6-1ce0-4de2-b059-4e4eb9ba0f37"}
{"level":"info","ts":"2026-07-13T16:07:07.253Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"0ce47ee6-1ce0-4de2-b059-4e4eb9ba0f37","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:07:08.222Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"a50aa1a3-1b56-4695-ad36-ce2d184eb4b0"}
{"level":"info","ts":"2026-07-13T16:07:08.222Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"a50aa1a3-1b56-4695-ad36-ce2d184eb4b0"}
{"level":"info","ts":"2026-07-13T16:07:08.222Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"a50aa1a3-1b56-4695-ad36-ce2d184eb4b0","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:07:08.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"a50aa1a3-1b56-4695-ad36-ce2d184eb4b0","Found 1 head Pod":"stop-b592q-head-wfqrz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:07:08.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"a50aa1a3-1b56-4695-ad36-ce2d184eb4b0","head Pod":"stop-b592q-head-wfqrz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-b592q-head-wfqrz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:07:08.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"a50aa1a3-1b56-4695-ad36-ce2d184eb4b0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:07:08.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"a50aa1a3-1b56-4695-ad36-ce2d184eb4b0","worker Pod":"stop-b592q-small-group-worker-9hklq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-b592q-small-group-worker-9hklq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:07:08.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"a50aa1a3-1b56-4695-ad36-ce2d184eb4b0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:07:08.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"a50aa1a3-1b56-4695-ad36-ce2d184eb4b0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:07:08.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"a50aa1a3-1b56-4695-ad36-ce2d184eb4b0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:07:08.222Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"a50aa1a3-1b56-4695-ad36-ce2d184eb4b0","podName":"stop-b592q-small-group-worker-9hklq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:07:08.222Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"a50aa1a3-1b56-4695-ad36-ce2d184eb4b0","podName":"stop-b592q-small-group-worker-9hklq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:07:08.222Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"a50aa1a3-1b56-4695-ad36-ce2d184eb4b0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:07:08.222Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"a50aa1a3-1b56-4695-ad36-ce2d184eb4b0","seconds":300}
{"level":"info","ts":"2026-07-13T16:07:10.253Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"83b8a4e5-ccf5-4df9-8aeb-7dbc1a81e446","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:10.253Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"83b8a4e5-ccf5-4df9-8aeb-7dbc1a81e446","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:10.253Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"83b8a4e5-ccf5-4df9-8aeb-7dbc1a81e446","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:10.253Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"83b8a4e5-ccf5-4df9-8aeb-7dbc1a81e446"}
{"level":"info","ts":"2026-07-13T16:07:10.253Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"83b8a4e5-ccf5-4df9-8aeb-7dbc1a81e446","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:07:13.255Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"63ef0879-9ac7-47d8-bf96-17615e5ae690","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:13.255Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"63ef0879-9ac7-47d8-bf96-17615e5ae690","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:13.255Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"63ef0879-9ac7-47d8-bf96-17615e5ae690","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:13.255Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"63ef0879-9ac7-47d8-bf96-17615e5ae690"}
{"level":"info","ts":"2026-07-13T16:07:13.255Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"63ef0879-9ac7-47d8-bf96-17615e5ae690","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:07:16.255Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6b9974f2-ed2c-4c53-962e-9c9169f5d045","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:16.255Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6b9974f2-ed2c-4c53-962e-9c9169f5d045","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:16.255Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6b9974f2-ed2c-4c53-962e-9c9169f5d045","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:16.255Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6b9974f2-ed2c-4c53-962e-9c9169f5d045"}
{"level":"info","ts":"2026-07-13T16:07:16.255Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6b9974f2-ed2c-4c53-962e-9c9169f5d045","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:07:18.263Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b50ff31a-0613-4544-bb94-ee0f6c2b0452"}
{"level":"info","ts":"2026-07-13T16:07:18.263Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b50ff31a-0613-4544-bb94-ee0f6c2b0452"}
{"level":"info","ts":"2026-07-13T16:07:18.263Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b50ff31a-0613-4544-bb94-ee0f6c2b0452","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:07:18.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b50ff31a-0613-4544-bb94-ee0f6c2b0452","Found 1 head Pod":"stop-b592q-head-wfqrz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:07:18.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b50ff31a-0613-4544-bb94-ee0f6c2b0452","head Pod":"stop-b592q-head-wfqrz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-b592q-head-wfqrz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:07:18.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b50ff31a-0613-4544-bb94-ee0f6c2b0452","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:07:18.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b50ff31a-0613-4544-bb94-ee0f6c2b0452","worker Pod":"stop-b592q-small-group-worker-9hklq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-b592q-small-group-worker-9hklq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:07:18.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b50ff31a-0613-4544-bb94-ee0f6c2b0452","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:07:18.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b50ff31a-0613-4544-bb94-ee0f6c2b0452","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:07:18.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b50ff31a-0613-4544-bb94-ee0f6c2b0452","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:07:18.263Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b50ff31a-0613-4544-bb94-ee0f6c2b0452","podName":"stop-b592q-small-group-worker-9hklq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:07:18.263Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b50ff31a-0613-4544-bb94-ee0f6c2b0452","podName":"stop-b592q-small-group-worker-9hklq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:07:18.263Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b50ff31a-0613-4544-bb94-ee0f6c2b0452","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:07:18.263Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b50ff31a-0613-4544-bb94-ee0f6c2b0452","seconds":300}
{"level":"info","ts":"2026-07-13T16:07:19.256Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"49bb2b42-b6c9-4518-9b90-934f963613b9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:19.256Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"49bb2b42-b6c9-4518-9b90-934f963613b9","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:19.256Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"49bb2b42-b6c9-4518-9b90-934f963613b9","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:19.256Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"49bb2b42-b6c9-4518-9b90-934f963613b9"}
{"level":"info","ts":"2026-07-13T16:07:19.256Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"49bb2b42-b6c9-4518-9b90-934f963613b9","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:07:19.273Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"deeb1325-64fe-4ad1-80f4-8d649b06bbc1"}
{"level":"info","ts":"2026-07-13T16:07:19.273Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"deeb1325-64fe-4ad1-80f4-8d649b06bbc1"}
{"level":"info","ts":"2026-07-13T16:07:19.273Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"deeb1325-64fe-4ad1-80f4-8d649b06bbc1","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:07:19.273Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"deeb1325-64fe-4ad1-80f4-8d649b06bbc1","Found 1 head Pod":"stop-b592q-head-wfqrz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:07:19.273Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"deeb1325-64fe-4ad1-80f4-8d649b06bbc1","head Pod":"stop-b592q-head-wfqrz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-b592q-head-wfqrz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:07:19.273Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"deeb1325-64fe-4ad1-80f4-8d649b06bbc1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:07:19.273Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"deeb1325-64fe-4ad1-80f4-8d649b06bbc1","worker Pod":"stop-b592q-small-group-worker-9hklq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-b592q-small-group-worker-9hklq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:07:19.273Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"deeb1325-64fe-4ad1-80f4-8d649b06bbc1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:07:19.273Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"deeb1325-64fe-4ad1-80f4-8d649b06bbc1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:07:19.273Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"deeb1325-64fe-4ad1-80f4-8d649b06bbc1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:07:19.273Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"deeb1325-64fe-4ad1-80f4-8d649b06bbc1","podName":"stop-b592q-small-group-worker-9hklq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:05:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:07:19.273Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"deeb1325-64fe-4ad1-80f4-8d649b06bbc1","podName":"stop-b592q-small-group-worker-9hklq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:05:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:07:19.273Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"deeb1325-64fe-4ad1-80f4-8d649b06bbc1","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T16:07:19.273Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"deeb1325-64fe-4ad1-80f4-8d649b06bbc1","name":"stop-b592q","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:06:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:07:19.286Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"6bc763fd-8497-430f-a0e4-0aea5a2d8ce8","rayCluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:19.286Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"65d217a2-2539-4d30-a4aa-eab36c050a3c","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:19.286Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"6bc763fd-8497-430f-a0e4-0aea5a2d8ce8","pod":"stop-b592q-head-wfqrz","ip":"10.133.0.36"}
{"level":"info","ts":"2026-07-13T16:07:19.286Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"36c29c38-6d3f-476c-a6f3-010386746931","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:19.286Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"36c29c38-6d3f-476c-a6f3-010386746931","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:19.286Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"36c29c38-6d3f-476c-a6f3-010386746931","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:19.286Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"6bc763fd-8497-430f-a0e4-0aea5a2d8ce8","pod":"stop-b592q-small-group-worker-9hklq","ip":"10.134.0.34"}
{"level":"info","ts":"2026-07-13T16:07:19.286Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"6bc763fd-8497-430f-a0e4-0aea5a2d8ce8","rayCluster":"stop-b592q","podIPs":["10.133.0.36","10.134.0.34"]}
{"level":"info","ts":"2026-07-13T16:07:19.286Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"36c29c38-6d3f-476c-a6f3-010386746931"}
{"level":"info","ts":"2026-07-13T16:07:19.286Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"36c29c38-6d3f-476c-a6f3-010386746931","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:07:19.302Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"65d217a2-2539-4d30-a4aa-eab36c050a3c","name":"stop-b592q-head"}
{"level":"info","ts":"2026-07-13T16:07:19.312Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"65d217a2-2539-4d30-a4aa-eab36c050a3c","name":"stop-b592q-workers"}
{"level":"info","ts":"2026-07-13T16:07:19.312Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"65d217a2-2539-4d30-a4aa-eab36c050a3c","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:21.286Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"93b1a155-fd1b-4cc8-8a20-e2df71c06322"}
{"level":"info","ts":"2026-07-13T16:07:21.286Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"93b1a155-fd1b-4cc8-8a20-e2df71c06322"}
{"level":"info","ts":"2026-07-13T16:07:21.286Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"93b1a155-fd1b-4cc8-8a20-e2df71c06322","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:07:21.286Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"93b1a155-fd1b-4cc8-8a20-e2df71c06322","Found 1 head Pod":"stop-b592q-head-wfqrz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:07:21.286Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"93b1a155-fd1b-4cc8-8a20-e2df71c06322","head Pod":"stop-b592q-head-wfqrz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-b592q-head-wfqrz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:07:21.286Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"93b1a155-fd1b-4cc8-8a20-e2df71c06322","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:07:21.286Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"93b1a155-fd1b-4cc8-8a20-e2df71c06322","worker Pod":"stop-b592q-small-group-worker-9hklq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-b592q-small-group-worker-9hklq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:07:21.286Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"93b1a155-fd1b-4cc8-8a20-e2df71c06322","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:07:21.286Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"93b1a155-fd1b-4cc8-8a20-e2df71c06322","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:07:21.286Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"93b1a155-fd1b-4cc8-8a20-e2df71c06322","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:07:21.286Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"93b1a155-fd1b-4cc8-8a20-e2df71c06322","podName":"stop-b592q-small-group-worker-9hklq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:05:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:07:21.286Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"93b1a155-fd1b-4cc8-8a20-e2df71c06322","podName":"stop-b592q-small-group-worker-9hklq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:05:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:07:21.286Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"93b1a155-fd1b-4cc8-8a20-e2df71c06322","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:07:21.286Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"93b1a155-fd1b-4cc8-8a20-e2df71c06322","seconds":300}
{"level":"info","ts":"2026-07-13T16:07:22.256Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"63ca9af9-ae63-448e-a69f-8e653fca5024","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:22.257Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"63ca9af9-ae63-448e-a69f-8e653fca5024","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:22.257Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"63ca9af9-ae63-448e-a69f-8e653fca5024","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:22.257Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"63ca9af9-ae63-448e-a69f-8e653fca5024"}
{"level":"info","ts":"2026-07-13T16:07:22.257Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"63ca9af9-ae63-448e-a69f-8e653fca5024","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:07:25.258Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7b31b88b-3681-438d-98da-0aab601dc2c6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:25.258Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7b31b88b-3681-438d-98da-0aab601dc2c6","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:25.258Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7b31b88b-3681-438d-98da-0aab601dc2c6","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:25.258Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7b31b88b-3681-438d-98da-0aab601dc2c6"}
{"level":"info","ts":"2026-07-13T16:07:25.258Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"7b31b88b-3681-438d-98da-0aab601dc2c6","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:07:28.258Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"517e0ee2-41a6-4a5f-9698-b7172ba4f917","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:28.258Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"517e0ee2-41a6-4a5f-9698-b7172ba4f917","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:28.258Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"517e0ee2-41a6-4a5f-9698-b7172ba4f917","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:28.258Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"517e0ee2-41a6-4a5f-9698-b7172ba4f917"}
{"level":"info","ts":"2026-07-13T16:07:28.258Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"517e0ee2-41a6-4a5f-9698-b7172ba4f917","RayCluster":"stop-b592q","State":""}
{"level":"info","ts":"2026-07-13T16:07:30.297Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"403c99ec-ab1f-430e-a259-27519466377f"}
{"level":"info","ts":"2026-07-13T16:07:30.297Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"403c99ec-ab1f-430e-a259-27519466377f"}
{"level":"info","ts":"2026-07-13T16:07:30.297Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"403c99ec-ab1f-430e-a259-27519466377f","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:07:30.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"403c99ec-ab1f-430e-a259-27519466377f","Found 1 head Pod":"stop-b592q-head-wfqrz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:07:30.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"403c99ec-ab1f-430e-a259-27519466377f","head Pod":"stop-b592q-head-wfqrz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-b592q-head-wfqrz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:07:30.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"403c99ec-ab1f-430e-a259-27519466377f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:07:30.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"403c99ec-ab1f-430e-a259-27519466377f","worker Pod":"stop-b592q-small-group-worker-9hklq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-b592q-small-group-worker-9hklq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:07:30.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"403c99ec-ab1f-430e-a259-27519466377f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:07:30.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"403c99ec-ab1f-430e-a259-27519466377f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:07:30.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"403c99ec-ab1f-430e-a259-27519466377f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:07:30.298Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"403c99ec-ab1f-430e-a259-27519466377f","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-13T16:07:30.298Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"403c99ec-ab1f-430e-a259-27519466377f","name":"stop-b592q","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:07:30.310Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"3b676536-00e3-4231-ba1e-af3c8633093e","rayCluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:30.310Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"72daf449-cf4f-4f2d-a4f0-699721804aaa","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:30.310Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5e66d003-8073-4ea9-a199-81d9fc6afecf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:30.310Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5e66d003-8073-4ea9-a199-81d9fc6afecf","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:30.310Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"3b676536-00e3-4231-ba1e-af3c8633093e","pod":"stop-b592q-head-wfqrz","ip":"10.133.0.36"}
{"level":"info","ts":"2026-07-13T16:07:30.310Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5e66d003-8073-4ea9-a199-81d9fc6afecf","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:30.310Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"3b676536-00e3-4231-ba1e-af3c8633093e","pod":"stop-b592q-small-group-worker-9hklq","ip":"10.134.0.34"}
{"level":"info","ts":"2026-07-13T16:07:30.310Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"3b676536-00e3-4231-ba1e-af3c8633093e","rayCluster":"stop-b592q","podIPs":["10.133.0.36","10.134.0.34"]}
{"level":"info","ts":"2026-07-13T16:07:30.310Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5e66d003-8073-4ea9-a199-81d9fc6afecf"}
{"level":"info","ts":"2026-07-13T16:07:30.310Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5e66d003-8073-4ea9-a199-81d9fc6afecf","head service name":"stop-b592q-head-svc","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:07:30.310Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5e66d003-8073-4ea9-a199-81d9fc6afecf","head service URL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-13T16:07:30.310Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5e66d003-8073-4ea9-a199-81d9fc6afecf"}
{"level":"info","ts":"2026-07-13T16:07:30.310Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5e66d003-8073-4ea9-a199-81d9fc6afecf","command":["if","!","ray","job","status","--address","http://stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","stop-dq2s6",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","--no-wait","--submission-id","stop-dq2s6","--","python /home/ray/jobs/stop.py",";","fi",";","ray","job","logs","--address","http://stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","--follow","stop-dq2s6"]}
{"level":"info","ts":"2026-07-13T16:07:30.318Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5e66d003-8073-4ea9-a199-81d9fc6afecf","Kubernetes Job":"stop"}
{"level":"info","ts":"2026-07-13T16:07:30.318Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5e66d003-8073-4ea9-a199-81d9fc6afecf","SubmissionMode":"K8sJobMode","RayCluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:30.318Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5e66d003-8073-4ea9-a199-81d9fc6afecf","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T16:07:30.318Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5e66d003-8073-4ea9-a199-81d9fc6afecf","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T16:07:30.325Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"72daf449-cf4f-4f2d-a4f0-699721804aaa","name":"stop-b592q-head"}
{"level":"info","ts":"2026-07-13T16:07:30.422Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"58543478-82a9-4c48-8a9c-79736c7aad79","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:30.422Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"58543478-82a9-4c48-8a9c-79736c7aad79","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:30.422Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"58543478-82a9-4c48-8a9c-79736c7aad79","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:30.422Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"58543478-82a9-4c48-8a9c-79736c7aad79"}
{"level":"info","ts":"2026-07-13T16:07:30.521Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"72daf449-cf4f-4f2d-a4f0-699721804aaa","name":"stop-b592q-workers"}
{"level":"info","ts":"2026-07-13T16:07:30.521Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"72daf449-cf4f-4f2d-a4f0-699721804aaa","cluster":"stop-b592q"}
{"level":"error","ts":"2026-07-13T16:07:30.527Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"58543478-82a9-4c48-8a9c-79736c7aad79","JobId":"stop-dq2s6","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:30.527Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"58543478-82a9-4c48-8a9c-79736c7aad79"}
{"level":"error","ts":"2026-07-13T16:07:30.527Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"58543478-82a9-4c48-8a9c-79736c7aad79","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:30.533Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"b949cc31-1042-42a0-9535-8b02f6032270","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:30.533Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"b949cc31-1042-42a0-9535-8b02f6032270","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:30.533Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"b949cc31-1042-42a0-9535-8b02f6032270","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:30.533Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"b949cc31-1042-42a0-9535-8b02f6032270"}
{"level":"error","ts":"2026-07-13T16:07:30.538Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"b949cc31-1042-42a0-9535-8b02f6032270","JobId":"stop-dq2s6","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:30.538Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"b949cc31-1042-42a0-9535-8b02f6032270"}
{"level":"error","ts":"2026-07-13T16:07:30.538Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"b949cc31-1042-42a0-9535-8b02f6032270","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:30.548Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"11ce8cb7-934e-4912-9122-50bc2f2d20c4","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:30.548Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"11ce8cb7-934e-4912-9122-50bc2f2d20c4","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:30.548Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"11ce8cb7-934e-4912-9122-50bc2f2d20c4","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:30.548Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"11ce8cb7-934e-4912-9122-50bc2f2d20c4"}
{"level":"error","ts":"2026-07-13T16:07:30.552Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"11ce8cb7-934e-4912-9122-50bc2f2d20c4","JobId":"stop-dq2s6","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:30.552Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"11ce8cb7-934e-4912-9122-50bc2f2d20c4"}
{"level":"error","ts":"2026-07-13T16:07:30.552Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"11ce8cb7-934e-4912-9122-50bc2f2d20c4","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:30.572Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"71092302-0a36-4acf-a505-0566696574e3","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:30.572Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"71092302-0a36-4acf-a505-0566696574e3","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:30.572Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"71092302-0a36-4acf-a505-0566696574e3","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:30.572Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"71092302-0a36-4acf-a505-0566696574e3"}
{"level":"error","ts":"2026-07-13T16:07:30.621Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"71092302-0a36-4acf-a505-0566696574e3","JobId":"stop-dq2s6","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:30.621Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"71092302-0a36-4acf-a505-0566696574e3"}
{"level":"error","ts":"2026-07-13T16:07:30.621Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"71092302-0a36-4acf-a505-0566696574e3","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:30.662Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"9aeeb7d1-af12-4d40-a70c-11080e3dc36d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:30.662Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"9aeeb7d1-af12-4d40-a70c-11080e3dc36d","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:30.662Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"9aeeb7d1-af12-4d40-a70c-11080e3dc36d","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:30.662Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"9aeeb7d1-af12-4d40-a70c-11080e3dc36d"}
{"level":"error","ts":"2026-07-13T16:07:30.667Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"9aeeb7d1-af12-4d40-a70c-11080e3dc36d","JobId":"stop-dq2s6","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:30.667Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"9aeeb7d1-af12-4d40-a70c-11080e3dc36d"}
{"level":"error","ts":"2026-07-13T16:07:30.667Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"9aeeb7d1-af12-4d40-a70c-11080e3dc36d","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:30.748Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d23cb2a2-ae0f-4399-bbe6-a5449ce1e442","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:30.748Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d23cb2a2-ae0f-4399-bbe6-a5449ce1e442","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:30.748Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d23cb2a2-ae0f-4399-bbe6-a5449ce1e442","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:30.748Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d23cb2a2-ae0f-4399-bbe6-a5449ce1e442"}
{"level":"error","ts":"2026-07-13T16:07:30.754Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d23cb2a2-ae0f-4399-bbe6-a5449ce1e442","JobId":"stop-dq2s6","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:30.754Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d23cb2a2-ae0f-4399-bbe6-a5449ce1e442"}
{"level":"error","ts":"2026-07-13T16:07:30.754Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d23cb2a2-ae0f-4399-bbe6-a5449ce1e442","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:30.915Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"3215d4a1-3f34-4ede-951f-1e8c094d33cb","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:30.915Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"3215d4a1-3f34-4ede-951f-1e8c094d33cb","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:30.915Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"3215d4a1-3f34-4ede-951f-1e8c094d33cb","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:30.915Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"3215d4a1-3f34-4ede-951f-1e8c094d33cb"}
{"level":"error","ts":"2026-07-13T16:07:30.920Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"3215d4a1-3f34-4ede-951f-1e8c094d33cb","JobId":"stop-dq2s6","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:30.920Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"3215d4a1-3f34-4ede-951f-1e8c094d33cb"}
{"level":"error","ts":"2026-07-13T16:07:30.920Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"3215d4a1-3f34-4ede-951f-1e8c094d33cb","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:31.240Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d10e1eb0-dbb1-426d-a3b6-b2548a3ffbb7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:31.240Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d10e1eb0-dbb1-426d-a3b6-b2548a3ffbb7","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:31.240Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d10e1eb0-dbb1-426d-a3b6-b2548a3ffbb7","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:31.241Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d10e1eb0-dbb1-426d-a3b6-b2548a3ffbb7"}
{"level":"error","ts":"2026-07-13T16:07:31.245Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d10e1eb0-dbb1-426d-a3b6-b2548a3ffbb7","JobId":"stop-dq2s6","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:31.246Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d10e1eb0-dbb1-426d-a3b6-b2548a3ffbb7"}
{"level":"error","ts":"2026-07-13T16:07:31.246Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d10e1eb0-dbb1-426d-a3b6-b2548a3ffbb7","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:31.341Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"283f4882-2c1d-4b3c-89e2-28fd7e898953","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:31.341Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"283f4882-2c1d-4b3c-89e2-28fd7e898953","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:31.341Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"283f4882-2c1d-4b3c-89e2-28fd7e898953","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:31.341Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"283f4882-2c1d-4b3c-89e2-28fd7e898953"}
{"level":"error","ts":"2026-07-13T16:07:31.345Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"283f4882-2c1d-4b3c-89e2-28fd7e898953","JobId":"stop-dq2s6","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:31.345Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"283f4882-2c1d-4b3c-89e2-28fd7e898953"}
{"level":"error","ts":"2026-07-13T16:07:31.345Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"283f4882-2c1d-4b3c-89e2-28fd7e898953","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:31.886Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a8685e50-b2c4-4453-b01c-535ac14f40f3","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:31.886Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a8685e50-b2c4-4453-b01c-535ac14f40f3","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:31.886Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a8685e50-b2c4-4453-b01c-535ac14f40f3","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:31.887Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a8685e50-b2c4-4453-b01c-535ac14f40f3"}
{"level":"error","ts":"2026-07-13T16:07:31.891Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a8685e50-b2c4-4453-b01c-535ac14f40f3","JobId":"stop-dq2s6","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:31.891Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a8685e50-b2c4-4453-b01c-535ac14f40f3"}
{"level":"error","ts":"2026-07-13T16:07:31.891Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"a8685e50-b2c4-4453-b01c-535ac14f40f3","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:32.310Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b836250d-71a8-4a06-88e0-0663343f1704"}
{"level":"info","ts":"2026-07-13T16:07:32.310Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b836250d-71a8-4a06-88e0-0663343f1704"}
{"level":"info","ts":"2026-07-13T16:07:32.310Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b836250d-71a8-4a06-88e0-0663343f1704","1 head service found":"stop-b592q-head-svc"}
{"level":"info","ts":"2026-07-13T16:07:32.310Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b836250d-71a8-4a06-88e0-0663343f1704","Found 1 head Pod":"stop-b592q-head-wfqrz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:07:32.310Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b836250d-71a8-4a06-88e0-0663343f1704","head Pod":"stop-b592q-head-wfqrz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-b592q-head-wfqrz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:07:32.310Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b836250d-71a8-4a06-88e0-0663343f1704","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:07:32.310Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b836250d-71a8-4a06-88e0-0663343f1704","worker Pod":"stop-b592q-small-group-worker-9hklq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-b592q-small-group-worker-9hklq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:07:32.310Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b836250d-71a8-4a06-88e0-0663343f1704","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:07:32.310Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b836250d-71a8-4a06-88e0-0663343f1704","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:07:32.310Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b836250d-71a8-4a06-88e0-0663343f1704","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:07:32.310Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b836250d-71a8-4a06-88e0-0663343f1704","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:07:32.310Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"b836250d-71a8-4a06-88e0-0663343f1704","seconds":300}
{"level":"info","ts":"2026-07-13T16:07:34.452Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2ad50364-6e60-42f6-b137-2cbd6fba67d5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:34.452Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2ad50364-6e60-42f6-b137-2cbd6fba67d5","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:34.452Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2ad50364-6e60-42f6-b137-2cbd6fba67d5","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:34.452Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2ad50364-6e60-42f6-b137-2cbd6fba67d5"}
{"level":"error","ts":"2026-07-13T16:07:34.457Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2ad50364-6e60-42f6-b137-2cbd6fba67d5","JobId":"stop-dq2s6","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:34.457Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2ad50364-6e60-42f6-b137-2cbd6fba67d5"}
{"level":"error","ts":"2026-07-13T16:07:34.457Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2ad50364-6e60-42f6-b137-2cbd6fba67d5","error":"Job stop-dq2s6 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:07:34.469Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"65ba8a06-03e5-44fc-a0a4-1c469893958b","rayCluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:34.469Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"65ba8a06-03e5-44fc-a0a4-1c469893958b","pod":"stop-b592q-head-wfqrz","ip":"10.133.0.36"}
{"level":"info","ts":"2026-07-13T16:07:34.469Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"65ba8a06-03e5-44fc-a0a4-1c469893958b","pod":"stop-b592q-small-group-worker-9hklq","ip":"10.134.0.34"}
{"level":"info","ts":"2026-07-13T16:07:34.469Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"65ba8a06-03e5-44fc-a0a4-1c469893958b","rayCluster":"stop-b592q","podIPs":["10.133.0.36","10.134.0.34"]}
{"level":"info","ts":"2026-07-13T16:07:34.948Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a04b1c43-f457-4e1c-8371-cceba3e04d84","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:07:34.948Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a04b1c43-f457-4e1c-8371-cceba3e04d84","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T16:07:34.948Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a04b1c43-f457-4e1c-8371-cceba3e04d84","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T16:07:34.948Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a04b1c43-f457-4e1c-8371-cceba3e04d84","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T16:07:39.578Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"04268bdc-f9cc-4edd-af91-9f619c232e96","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:39.578Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"04268bdc-f9cc-4edd-af91-9f619c232e96","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:39.578Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"04268bdc-f9cc-4edd-af91-9f619c232e96","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:39.578Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"04268bdc-f9cc-4edd-af91-9f619c232e96"}
{"level":"info","ts":"2026-07-13T16:07:39.583Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"04268bdc-f9cc-4edd-af91-9f619c232e96","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:07:39.583Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"04268bdc-f9cc-4edd-af91-9f619c232e96","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T16:07:39.596Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"446b9519-f2f5-4e7f-8664-5f002808deca","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:39.596Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"446b9519-f2f5-4e7f-8664-5f002808deca","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:39.597Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"446b9519-f2f5-4e7f-8664-5f002808deca","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:39.597Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"446b9519-f2f5-4e7f-8664-5f002808deca"}
{"level":"info","ts":"2026-07-13T16:07:39.599Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"446b9519-f2f5-4e7f-8664-5f002808deca","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:07:39.599Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"446b9519-f2f5-4e7f-8664-5f002808deca","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T16:07:39.612Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"98d9bf4f-616e-440e-a9ea-fb7dc8bf2437","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:39.612Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"98d9bf4f-616e-440e-a9ea-fb7dc8bf2437","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:39.612Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"98d9bf4f-616e-440e-a9ea-fb7dc8bf2437","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:39.612Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"98d9bf4f-616e-440e-a9ea-fb7dc8bf2437"}
{"level":"info","ts":"2026-07-13T16:07:39.615Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"98d9bf4f-616e-440e-a9ea-fb7dc8bf2437","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:07:42.596Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"068ba259-1336-4b97-99aa-3ca2f82045bb","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:42.596Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"068ba259-1336-4b97-99aa-3ca2f82045bb","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:42.596Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"068ba259-1336-4b97-99aa-3ca2f82045bb","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:42.596Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"068ba259-1336-4b97-99aa-3ca2f82045bb"}
{"level":"info","ts":"2026-07-13T16:07:42.599Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"068ba259-1336-4b97-99aa-3ca2f82045bb","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:07:45.599Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"f5379226-c732-40e1-b831-5a6abac2a467","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:45.599Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"f5379226-c732-40e1-b831-5a6abac2a467","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:45.599Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"f5379226-c732-40e1-b831-5a6abac2a467","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:45.599Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"f5379226-c732-40e1-b831-5a6abac2a467"}
{"level":"info","ts":"2026-07-13T16:07:45.602Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"f5379226-c732-40e1-b831-5a6abac2a467","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:07:48.603Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"e6250b36-8dff-4db5-991c-05d5632423bc","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:48.603Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"e6250b36-8dff-4db5-991c-05d5632423bc","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:48.603Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"e6250b36-8dff-4db5-991c-05d5632423bc","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:48.603Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"e6250b36-8dff-4db5-991c-05d5632423bc"}
{"level":"info","ts":"2026-07-13T16:07:48.606Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"e6250b36-8dff-4db5-991c-05d5632423bc","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:07:51.606Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"868677cc-b0df-4987-8e5c-4518cdd0f7f2","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:51.606Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"868677cc-b0df-4987-8e5c-4518cdd0f7f2","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:51.606Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"868677cc-b0df-4987-8e5c-4518cdd0f7f2","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:51.606Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"868677cc-b0df-4987-8e5c-4518cdd0f7f2"}
{"level":"info","ts":"2026-07-13T16:07:51.609Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"868677cc-b0df-4987-8e5c-4518cdd0f7f2","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:07:54.610Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ca532bb0-0e32-40c2-ab21-70d2a8ef4d35","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:54.610Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ca532bb0-0e32-40c2-ab21-70d2a8ef4d35","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:54.610Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ca532bb0-0e32-40c2-ab21-70d2a8ef4d35","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:54.610Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ca532bb0-0e32-40c2-ab21-70d2a8ef4d35"}
{"level":"info","ts":"2026-07-13T16:07:54.613Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"ca532bb0-0e32-40c2-ab21-70d2a8ef4d35","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:07:57.614Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"19f6dcb3-b060-4593-a158-aa8001221514","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:07:57.614Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"19f6dcb3-b060-4593-a158-aa8001221514","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:07:57.614Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"19f6dcb3-b060-4593-a158-aa8001221514","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:07:57.614Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"19f6dcb3-b060-4593-a158-aa8001221514"}
{"level":"info","ts":"2026-07-13T16:07:57.617Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"19f6dcb3-b060-4593-a158-aa8001221514","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:08:00.617Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"27bf8fc8-7985-4e11-8038-c493d3b27490","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:00.617Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"27bf8fc8-7985-4e11-8038-c493d3b27490","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:08:00.617Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"27bf8fc8-7985-4e11-8038-c493d3b27490","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:00.617Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"27bf8fc8-7985-4e11-8038-c493d3b27490"}
{"level":"info","ts":"2026-07-13T16:08:00.620Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"27bf8fc8-7985-4e11-8038-c493d3b27490","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z","endTime":"2026-07-13T16:08:00Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:08:00.620Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"27bf8fc8-7985-4e11-8038-c493d3b27490","old JobStatus":"RUNNING","new JobStatus":"STOPPED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T16:08:00.636Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5f3da0c4-3c39-40e2-86de-fb0254239566","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:00.636Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5f3da0c4-3c39-40e2-86de-fb0254239566","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:08:00.636Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5f3da0c4-3c39-40e2-86de-fb0254239566","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:00.637Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5f3da0c4-3c39-40e2-86de-fb0254239566"}
{"level":"info","ts":"2026-07-13T16:08:00.639Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"5f3da0c4-3c39-40e2-86de-fb0254239566","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z","endTime":"2026-07-13T16:08:00Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z","endTime":"2026-07-13T16:08:00Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:08:03.636Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6d1bea31-8b29-40bc-9d7d-9d78574e52b2","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:03.636Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6d1bea31-8b29-40bc-9d7d-9d78574e52b2","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:08:03.636Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6d1bea31-8b29-40bc-9d7d-9d78574e52b2","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:03.636Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6d1bea31-8b29-40bc-9d7d-9d78574e52b2"}
{"level":"info","ts":"2026-07-13T16:08:03.639Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6d1bea31-8b29-40bc-9d7d-9d78574e52b2","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z","endTime":"2026-07-13T16:08:00Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z","endTime":"2026-07-13T16:08:00Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:08:06.073Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"fafdd451-2576-4ebf-a857-88ddd390894b","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:06.073Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"fafdd451-2576-4ebf-a857-88ddd390894b","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:08:06.073Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"fafdd451-2576-4ebf-a857-88ddd390894b","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:06.073Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"fafdd451-2576-4ebf-a857-88ddd390894b"}
{"level":"info","ts":"2026-07-13T16:08:06.076Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"fafdd451-2576-4ebf-a857-88ddd390894b","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z","endTime":"2026-07-13T16:08:00Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z","endTime":"2026-07-13T16:08:00Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:08:06.639Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"8f14d912-d27d-497c-bf0a-e69667c94b07","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:06.639Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"8f14d912-d27d-497c-bf0a-e69667c94b07","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:08:06.640Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"8f14d912-d27d-497c-bf0a-e69667c94b07","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:06.640Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"8f14d912-d27d-497c-bf0a-e69667c94b07"}
{"level":"info","ts":"2026-07-13T16:08:06.642Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"8f14d912-d27d-497c-bf0a-e69667c94b07","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z","endTime":"2026-07-13T16:08:00Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z","endTime":"2026-07-13T16:08:00Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:08:07.081Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6146b228-856d-4d7a-a3e0-f4946bfd33ae","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:07.081Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6146b228-856d-4d7a-a3e0-f4946bfd33ae","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:08:07.081Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6146b228-856d-4d7a-a3e0-f4946bfd33ae","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:07.081Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6146b228-856d-4d7a-a3e0-f4946bfd33ae"}
{"level":"info","ts":"2026-07-13T16:08:07.084Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6146b228-856d-4d7a-a3e0-f4946bfd33ae","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z","endTime":"2026-07-13T16:08:00Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z","endTime":"2026-07-13T16:08:00Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:08:07.099Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"75304631-79a5-42d7-92ed-264a9939ce8e","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:07.099Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"75304631-79a5-42d7-92ed-264a9939ce8e","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:08:07.099Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"75304631-79a5-42d7-92ed-264a9939ce8e","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:07.099Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"75304631-79a5-42d7-92ed-264a9939ce8e"}
{"level":"info","ts":"2026-07-13T16:08:07.102Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"75304631-79a5-42d7-92ed-264a9939ce8e","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z","endTime":"2026-07-13T16:08:00Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:07:38Z","endTime":"2026-07-13T16:08:00Z"},"jobId":"stop-dq2s6","rayClusterName":"stop-b592q","dashboardURL":"stop-b592q-head-svc.test-ns-x5qvg.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Complete","message":"Job was intentionally stopped.","startTime":"2026-07-13T16:05:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:07:30Z","stateTransitionTimes":{"ready":"2026-07-13T16:07:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.36","serviceIP":"10.133.0.36","podName":"stop-b592q-head-wfqrz","serviceName":"stop-b592q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:06:58Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:07:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:05:34Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:05:34Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:08:07.102Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"75304631-79a5-42d7-92ed-264a9939ce8e","old JobStatus":"STOPPED","new JobStatus":"STOPPED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-07-13T16:08:07.115Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d0f00e8b-6bae-4577-a71d-73d3d53210fc","JobStatus":"STOPPED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:07.115Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"d0f00e8b-6bae-4577-a71d-73d3d53210fc","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 16:08:07 +0000 UTC","Now":"2026-07-13T16:08:07.115Z","ShutdownTime":"2026-07-13T16:08:07.000Z"}
{"level":"info","ts":"2026-07-13T16:08:07.886Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"3364133a-f6d6-449d-aadf-7ff9bab08079","DeletionTimestamp":"2026-07-13 16:08:07 +0000 UTC"}
{"level":"info","ts":"2026-07-13T16:08:07.886Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"3364133a-f6d6-449d-aadf-7ff9bab08079","finalizer":"ray.io/rayjob-finalizer"}
{"level":"error","ts":"2026-07-13T16:08:07.904Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"8bc0e578-f2b5-4c2d-8afa-e84fdec28e17","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:07.904Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"8bc0e578-f2b5-4c2d-8afa-e84fdec28e17"}
{"level":"error","ts":"2026-07-13T16:08:07.904Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"8bc0e578-f2b5-4c2d-8afa-e84fdec28e17","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:07.905Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"4d96ee09-3ffb-49d6-b422-f7726ecc5286"}
{"level":"error","ts":"2026-07-13T16:08:07.909Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"49aec359-6aa1-46dd-8ebc-3f82e688d873","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:07.909Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"49aec359-6aa1-46dd-8ebc-3f82e688d873"}
{"level":"error","ts":"2026-07-13T16:08:07.910Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"49aec359-6aa1-46dd-8ebc-3f82e688d873","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:08:07.920Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"dc50d790-5cfd-484e-bff8-65bc440d3039","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:07.920Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"dc50d790-5cfd-484e-bff8-65bc440d3039"}
{"level":"error","ts":"2026-07-13T16:08:07.920Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"dc50d790-5cfd-484e-bff8-65bc440d3039","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:07.934Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"3091adff-918d-49b6-b921-2de6b51d714c","namespacedName":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:07.934Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"6e7545d2-32dc-4f5c-8d6e-fc5eaccd3ebe","rayCluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:08:07.934Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"3091adff-918d-49b6-b921-2de6b51d714c","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:08:07.934Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"3091adff-918d-49b6-b921-2de6b51d714c","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:08:07.934Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"f59abcb5-604d-4c7b-83db-e5363bd8ced9"}
{"level":"info","ts":"2026-07-13T16:08:07.934Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"43cc2ca2-2d71-421d-9182-eab8e6881b50"}
{"level":"info","ts":"2026-07-13T16:08:07.934Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"3091adff-918d-49b6-b921-2de6b51d714c","namespace":"test-ns-x5qvg","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-13T16:08:07.934Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"6e7545d2-32dc-4f5c-8d6e-fc5eaccd3ebe","pod":"stop-b592q-head-wfqrz","ip":"10.133.0.36"}
{"level":"info","ts":"2026-07-13T16:08:07.934Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"6e7545d2-32dc-4f5c-8d6e-fc5eaccd3ebe","pod":"stop-b592q-small-group-worker-9hklq","ip":"10.134.0.34"}
{"level":"info","ts":"2026-07-13T16:08:07.934Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"6e7545d2-32dc-4f5c-8d6e-fc5eaccd3ebe","rayCluster":"stop-b592q","podIPs":["10.133.0.36","10.134.0.34"]}
{"level":"info","ts":"2026-07-13T16:08:07.934Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"reconcileID":"5bdf3f95-6a84-4db2-b478-1f0694f17ccc"}
{"level":"info","ts":"2026-07-13T16:08:07.936Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"2b099629-0e3a-48bc-a698-c844a01b4195"}
{"level":"error","ts":"2026-07-13T16:08:07.940Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"8f717e19-a3fc-48ba-825b-4d439848d70e","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:07.940Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"8f717e19-a3fc-48ba-825b-4d439848d70e"}
{"level":"error","ts":"2026-07-13T16:08:07.940Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"8f717e19-a3fc-48ba-825b-4d439848d70e","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:07.941Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"3091adff-918d-49b6-b921-2de6b51d714c","name":"kuberay-gateway-access","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:08:07.952Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"3091adff-918d-49b6-b921-2de6b51d714c","name":"test-ns-x5qvg-stop-b592q","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-13T16:08:07.959Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"3091adff-918d-49b6-b921-2de6b51d714c","configMap":"kube-rbac-proxy-config-stop-b592q"}
{"level":"info","ts":"2026-07-13T16:08:07.965Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"3091adff-918d-49b6-b921-2de6b51d714c","serviceAccount":"stop-b592q-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"error","ts":"2026-07-13T16:08:07.981Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"86b8ac7f-b5c0-43ad-985b-47b1045bbd8a","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:07.981Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"86b8ac7f-b5c0-43ad-985b-47b1045bbd8a"}
{"level":"error","ts":"2026-07-13T16:08:07.981Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"86b8ac7f-b5c0-43ad-985b-47b1045bbd8a","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:07.991Z","logger":"raycluster-resource","msg":"default","name":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:08:07.991Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-b592q","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:08:08.000Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"3091adff-918d-49b6-b921-2de6b51d714c","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:08:08.000Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"785c98f5-6b59-4d6f-bf6b-35e1aae9d35b","namespacedName":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:08.001Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"785c98f5-6b59-4d6f-bf6b-35e1aae9d35b","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:08:08.001Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"785c98f5-6b59-4d6f-bf6b-35e1aae9d35b","cluster":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:08:08.001Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"785c98f5-6b59-4d6f-bf6b-35e1aae9d35b","namespace":"test-ns-x5qvg","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-13T16:08:08.001Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"785c98f5-6b59-4d6f-bf6b-35e1aae9d35b","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-13T16:08:08.001Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"3d0f4946-f970-49d0-bf7c-4817302f7763","namespace":"test-ns-x5qvg","clusterName":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:08:08.001Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"d167d498-9070-4d6f-9039-1b156c9e381a"}
{"level":"info","ts":"2026-07-13T16:08:08.001Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"92fe66c5-9a57-4f83-ae53-f0aa00a9bf68"}
{"level":"info","ts":"2026-07-13T16:08:08.009Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"c128a5c8-9fe6-48ba-9e58-79096bb07142"}
{"level":"info","ts":"2026-07-13T16:08:08.021Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"025d844f-d1e7-48d1-8d61-a83d877a34be"}
{"level":"error","ts":"2026-07-13T16:08:08.026Z","msg":"Failed to delete CA issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"3d0f4946-f970-49d0-bf7c-4817302f7763","issuer":"ray-ca-issuer-stop-b592q","error":"issuers.cert-manager.io \"ray-ca-issuer-stop-b592q\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:247\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:08:08.026Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"785c98f5-6b59-4d6f-bf6b-35e1aae9d35b","error":"rayclusters.ray.io \"stop-b592q\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:08.027Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"785c98f5-6b59-4d6f-bf6b-35e1aae9d35b"}
{"level":"error","ts":"2026-07-13T16:08:08.027Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"785c98f5-6b59-4d6f-bf6b-35e1aae9d35b","error":"rayclusters.ray.io \"stop-b592q\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:08.027Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"c801b5d9-0220-4fc8-9f1a-0e2184e2d0ef","namespacedName":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:08.027Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"c801b5d9-0220-4fc8-9f1a-0e2184e2d0ef"}
{"level":"info","ts":"2026-07-13T16:08:08.027Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"c801b5d9-0220-4fc8-9f1a-0e2184e2d0ef","namespace":"test-ns-x5qvg","authClusters":0}
{"level":"info","ts":"2026-07-13T16:08:08.027Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"c801b5d9-0220-4fc8-9f1a-0e2184e2d0ef","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:08:08.032Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"ee81cd41-a73a-43d7-9f91-dce4e133dc00","namespacedName":{"name":"stop-b592q","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:08.032Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"ee81cd41-a73a-43d7-9f91-dce4e133dc00"}
{"level":"info","ts":"2026-07-13T16:08:08.032Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"ee81cd41-a73a-43d7-9f91-dce4e133dc00","namespace":"test-ns-x5qvg","authClusters":0}
{"level":"info","ts":"2026-07-13T16:08:08.032Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"ee81cd41-a73a-43d7-9f91-dce4e133dc00","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:08:08.049Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"3d0f4946-f970-49d0-bf7c-4817302f7763","errorCount":1}
{"level":"info","ts":"2026-07-13T16:08:08.049Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"3d0f4946-f970-49d0-bf7c-4817302f7763"}
{"level":"error","ts":"2026-07-13T16:08:08.049Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"3d0f4946-f970-49d0-bf7c-4817302f7763","error":"issuers.cert-manager.io \"ray-ca-issuer-stop-b592q\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:08.054Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"a607a731-01e9-4a51-9e77-6dcc01a3c46f","namespace":"test-ns-x5qvg","clusterName":"stop-b592q"}
{"level":"info","ts":"2026-07-13T16:08:08.055Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-b592q","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"stop-b592q","reconcileID":"a607a731-01e9-4a51-9e77-6dcc01a3c46f","namespace":"test-ns-x5qvg","clusterName":"stop-b592q"}
{"level":"error","ts":"2026-07-13T16:08:08.062Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"6cf90a6f-7d33-4393-b5a5-d7cf8c1b353e","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:08.062Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"6cf90a6f-7d33-4393-b5a5-d7cf8c1b353e"}
{"level":"error","ts":"2026-07-13T16:08:08.062Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"6cf90a6f-7d33-4393-b5a5-d7cf8c1b353e","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:08:08.222Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"87bd8320-7de4-4284-83e1-0341a2826574","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:08.222Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"87bd8320-7de4-4284-83e1-0341a2826574"}
{"level":"error","ts":"2026-07-13T16:08:08.222Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"87bd8320-7de4-4284-83e1-0341a2826574","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:08:08.543Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"33fbc3e4-a99e-4809-8e34-6708ce7cfa8e","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:08.543Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"33fbc3e4-a99e-4809-8e34-6708ce7cfa8e"}
{"level":"error","ts":"2026-07-13T16:08:08.543Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"33fbc3e4-a99e-4809-8e34-6708ce7cfa8e","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:08:09.184Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"a44b097b-2882-46a3-8d22-f1eb77cea739","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:09.184Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"a44b097b-2882-46a3-8d22-f1eb77cea739"}
{"level":"error","ts":"2026-07-13T16:08:09.184Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"a44b097b-2882-46a3-8d22-f1eb77cea739","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:09.643Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-x5qvg"},"reconcileID":"6af246bf-d19e-444d-8714-a3bc7750ef13"}
{"level":"error","ts":"2026-07-13T16:08:10.464Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"73d6d3b6-8144-46d9-9447-436014600631","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:10.464Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"73d6d3b6-8144-46d9-9447-436014600631"}
{"level":"error","ts":"2026-07-13T16:08:10.464Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"73d6d3b6-8144-46d9-9447-436014600631","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:12.909Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"ef108c14-c285-4486-bd68-bbe5e8661f49","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:12.909Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"ef108c14-c285-4486-bd68-bbe5e8661f49","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T16:08:12.920Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"ef108c14-c285-4486-bd68-bbe5e8661f49"}
{"level":"info","ts":"2026-07-13T16:08:12.920Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"ef108c14-c285-4486-bd68-bbe5e8661f49","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-13T16:08:12.920Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"ef108c14-c285-4486-bd68-bbe5e8661f49","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-sfgvz","rayClusterName":"long-running-rfrb2","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T16:08:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T16:08:12.920Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"ef108c14-c285-4486-bd68-bbe5e8661f49","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T16:08:12.930Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"79e54856-1a69-440d-a12b-cce21e25d6ff","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:12.930Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"79e54856-1a69-440d-a12b-cce21e25d6ff","name":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:12.930Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"79e54856-1a69-440d-a12b-cce21e25d6ff","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:12.930Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"79e54856-1a69-440d-a12b-cce21e25d6ff","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:12.934Z","logger":"raycluster-resource","msg":"default","name":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:12.935Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:08:12.949Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"ce4ab659-34bc-48fd-9277-5ee1d452dd1e","namespacedName":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:12.949Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"48efa98e-a936-4108-8dde-5bec28e25ce6","rayCluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:12.949Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"48efa98e-a936-4108-8dde-5bec28e25ce6","rayCluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:12.949Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"ce4ab659-34bc-48fd-9277-5ee1d452dd1e","mode":"IntegratedOAuth","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:12.949Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"ce4ab659-34bc-48fd-9277-5ee1d452dd1e","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:12.949Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"ce4ab659-34bc-48fd-9277-5ee1d452dd1e","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:12.949Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"88a0de0a-b3e7-4ddf-9abe-4fc624d96a50","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:12.949Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"9740865b-d98b-4c79-a836-f5ab7ae2e003"}
{"level":"info","ts":"2026-07-13T16:08:12.949Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"9740865b-d98b-4c79-a836-f5ab7ae2e003"}
{"level":"info","ts":"2026-07-13T16:08:12.949Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"79e54856-1a69-440d-a12b-cce21e25d6ff","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:12.949Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"79e54856-1a69-440d-a12b-cce21e25d6ff"}
{"level":"info","ts":"2026-07-13T16:08:12.949Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"79e54856-1a69-440d-a12b-cce21e25d6ff","RayCluster":"long-running-rfrb2","State":""}
{"level":"info","ts":"2026-07-13T16:08:12.949Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"1d58e895-3b3c-404a-b459-5781537c21fe","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:12.949Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"1d58e895-3b3c-404a-b459-5781537c21fe","name":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:12.949Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"1d58e895-3b3c-404a-b459-5781537c21fe","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:12.949Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"1d58e895-3b3c-404a-b459-5781537c21fe"}
{"level":"info","ts":"2026-07-13T16:08:12.949Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"1d58e895-3b3c-404a-b459-5781537c21fe","RayCluster":"long-running-rfrb2","State":""}
{"level":"info","ts":"2026-07-13T16:08:12.954Z","logger":"raycluster-resource","msg":"default","name":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:12.954Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:08:12.957Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"9740865b-d98b-4c79-a836-f5ab7ae2e003","name":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:12.957Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"9740865b-d98b-4c79-a836-f5ab7ae2e003","error":"head certificate secret ray-head-secret-long-running-rfrb2 not found"}
{"level":"error","ts":"2026-07-13T16:08:12.957Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"9740865b-d98b-4c79-a836-f5ab7ae2e003","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:12.957Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"9740865b-d98b-4c79-a836-f5ab7ae2e003","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found"}
{"level":"info","ts":"2026-07-13T16:08:12.957Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"9740865b-d98b-4c79-a836-f5ab7ae2e003","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:12.957Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"9740865b-d98b-4c79-a836-f5ab7ae2e003","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:12.957Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"9740865b-d98b-4c79-a836-f5ab7ae2e003","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:12.957Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"9740865b-d98b-4c79-a836-f5ab7ae2e003","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T16:08:12.957Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"9740865b-d98b-4c79-a836-f5ab7ae2e003","name":"long-running-rfrb2","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:08:12.966Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"88a0de0a-b3e7-4ddf-9abe-4fc624d96a50","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:12.966Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"6798e219-688b-4997-8472-3de4a62ed3c1","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:12.967Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"64969120-5255-4ede-9bcc-034b43a44357","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:12.967Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"64969120-5255-4ede-9bcc-034b43a44357","name":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:12.967Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"64969120-5255-4ede-9bcc-034b43a44357","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:12.967Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"64969120-5255-4ede-9bcc-034b43a44357"}
{"level":"info","ts":"2026-07-13T16:08:12.967Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"64969120-5255-4ede-9bcc-034b43a44357","RayCluster":"long-running-rfrb2","State":""}
{"level":"info","ts":"2026-07-13T16:08:12.970Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"9740865b-d98b-4c79-a836-f5ab7ae2e003","name":"long-running-rfrb2","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-rfrb2\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-x5qvg","name":"long-running-rfrb2"}}
{"level":"info","ts":"2026-07-13T16:08:12.970Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"9740865b-d98b-4c79-a836-f5ab7ae2e003"}
{"level":"error","ts":"2026-07-13T16:08:12.970Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"9740865b-d98b-4c79-a836-f5ab7ae2e003","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:12.970Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ec9a74e2-512c-4027-b03d-9e398786e511"}
{"level":"info","ts":"2026-07-13T16:08:12.970Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ec9a74e2-512c-4027-b03d-9e398786e511"}
{"level":"info","ts":"2026-07-13T16:08:12.970Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ec9a74e2-512c-4027-b03d-9e398786e511","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:12.970Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ec9a74e2-512c-4027-b03d-9e398786e511","error":"head certificate secret ray-head-secret-long-running-rfrb2 not found"}
{"level":"error","ts":"2026-07-13T16:08:12.970Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ec9a74e2-512c-4027-b03d-9e398786e511","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:12.970Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ec9a74e2-512c-4027-b03d-9e398786e511","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found"}
{"level":"info","ts":"2026-07-13T16:08:12.970Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ec9a74e2-512c-4027-b03d-9e398786e511","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:12.970Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ec9a74e2-512c-4027-b03d-9e398786e511","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:12.970Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ec9a74e2-512c-4027-b03d-9e398786e511","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:12.970Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ec9a74e2-512c-4027-b03d-9e398786e511","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T16:08:12.970Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ec9a74e2-512c-4027-b03d-9e398786e511","name":"long-running-rfrb2","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:08:12.974Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"ce4ab659-34bc-48fd-9277-5ee1d452dd1e","name":"long-running-rfrb2-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:08:12.974Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"48efa98e-a936-4108-8dde-5bec28e25ce6","rayCluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:12.974Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"48efa98e-a936-4108-8dde-5bec28e25ce6","rayCluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:13.021Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"6798e219-688b-4997-8472-3de4a62ed3c1","name":"long-running-rfrb2-head"}
{"level":"info","ts":"2026-07-13T16:08:13.021Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ec9a74e2-512c-4027-b03d-9e398786e511"}
{"level":"info","ts":"2026-07-13T16:08:13.021Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"48efa98e-a936-4108-8dde-5bec28e25ce6","rayCluster":"long-running-rfrb2","secret":"ca-secret"}
{"level":"error","ts":"2026-07-13T16:08:13.021Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ec9a74e2-512c-4027-b03d-9e398786e511","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:13.021Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"ce4ab659-34bc-48fd-9277-5ee1d452dd1e","name":"kuberay-gateway-access","namespace":"test-ns-x5qvg","operation":"created"}
{"level":"info","ts":"2026-07-13T16:08:13.022Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8cfb5dcb-0dcc-40d7-8400-da5293b06201"}
{"level":"info","ts":"2026-07-13T16:08:13.021Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"48efa98e-a936-4108-8dde-5bec28e25ce6","rayCluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:13.022Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8cfb5dcb-0dcc-40d7-8400-da5293b06201"}
{"level":"info","ts":"2026-07-13T16:08:13.022Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8cfb5dcb-0dcc-40d7-8400-da5293b06201","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:13.022Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8cfb5dcb-0dcc-40d7-8400-da5293b06201","error":"head certificate secret ray-head-secret-long-running-rfrb2 not found"}
{"level":"error","ts":"2026-07-13T16:08:13.022Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8cfb5dcb-0dcc-40d7-8400-da5293b06201","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:13.221Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8cfb5dcb-0dcc-40d7-8400-da5293b06201","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found"}
{"level":"info","ts":"2026-07-13T16:08:13.023Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"6c6d027a-78ad-4e87-af1f-f18d5dbe604e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:13.221Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"6c6d027a-78ad-4e87-af1f-f18d5dbe604e","name":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:13.122Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"ce4ab659-34bc-48fd-9277-5ee1d452dd1e","name":"test-ns-x5qvg-long-running-rfrb2","operation":"created"}
{"level":"info","ts":"2026-07-13T16:08:13.221Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"6c6d027a-78ad-4e87-af1f-f18d5dbe604e","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:13.221Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8cfb5dcb-0dcc-40d7-8400-da5293b06201","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:13.122Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"48efa98e-a936-4108-8dde-5bec28e25ce6","rayCluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:13.221Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8cfb5dcb-0dcc-40d7-8400-da5293b06201","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:13.221Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8cfb5dcb-0dcc-40d7-8400-da5293b06201","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:13.221Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8cfb5dcb-0dcc-40d7-8400-da5293b06201","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T16:08:13.221Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8cfb5dcb-0dcc-40d7-8400-da5293b06201","name":"long-running-rfrb2","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:13Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:13Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:13Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:08:13.221Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"6c6d027a-78ad-4e87-af1f-f18d5dbe604e"}
{"level":"info","ts":"2026-07-13T16:08:13.221Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"6c6d027a-78ad-4e87-af1f-f18d5dbe604e","RayCluster":"long-running-rfrb2","State":""}
{"level":"error","ts":"2026-07-13T16:08:13.221Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"5e34a528-d48f-4741-adfd-9e85e28dcc7f","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:13.125Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"6798e219-688b-4997-8472-3de4a62ed3c1","name":"long-running-rfrb2-workers"}
{"level":"info","ts":"2026-07-13T16:08:13.222Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"6798e219-688b-4997-8472-3de4a62ed3c1","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:13.222Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"c56cc1a9-ae05-4b54-9229-2e89ab6758e5","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:13.222Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"5e34a528-d48f-4741-adfd-9e85e28dcc7f"}
{"level":"error","ts":"2026-07-13T16:08:13.222Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"5e34a528-d48f-4741-adfd-9e85e28dcc7f","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:13.228Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"ce4ab659-34bc-48fd-9277-5ee1d452dd1e","name":"kube-rbac-proxy-config-long-running-rfrb2","operation":"created"}
{"level":"info","ts":"2026-07-13T16:08:13.233Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"48efa98e-a936-4108-8dde-5bec28e25ce6","rayCluster":"long-running-rfrb2","podIPs":[]}
{"level":"info","ts":"2026-07-13T16:08:13.233Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8cfb5dcb-0dcc-40d7-8400-da5293b06201","name":"long-running-rfrb2","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-rfrb2\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-x5qvg","name":"long-running-rfrb2"}}
{"level":"info","ts":"2026-07-13T16:08:13.233Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8cfb5dcb-0dcc-40d7-8400-da5293b06201"}
{"level":"error","ts":"2026-07-13T16:08:13.233Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8cfb5dcb-0dcc-40d7-8400-da5293b06201","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:13.234Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b6c6c720-88dd-457d-81dd-2abbc5edaaf4"}
{"level":"info","ts":"2026-07-13T16:08:13.234Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b6c6c720-88dd-457d-81dd-2abbc5edaaf4"}
{"level":"info","ts":"2026-07-13T16:08:13.234Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b6c6c720-88dd-457d-81dd-2abbc5edaaf4","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:13.234Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b6c6c720-88dd-457d-81dd-2abbc5edaaf4","error":"head certificate secret ray-head-secret-long-running-rfrb2 not found"}
{"level":"error","ts":"2026-07-13T16:08:13.234Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b6c6c720-88dd-457d-81dd-2abbc5edaaf4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:13.234Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b6c6c720-88dd-457d-81dd-2abbc5edaaf4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found"}
{"level":"info","ts":"2026-07-13T16:08:13.234Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b6c6c720-88dd-457d-81dd-2abbc5edaaf4","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:13.234Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b6c6c720-88dd-457d-81dd-2abbc5edaaf4","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:13.234Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b6c6c720-88dd-457d-81dd-2abbc5edaaf4","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:13.234Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b6c6c720-88dd-457d-81dd-2abbc5edaaf4"}
{"level":"error","ts":"2026-07-13T16:08:13.234Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b6c6c720-88dd-457d-81dd-2abbc5edaaf4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:08:13.237Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"ce4ab659-34bc-48fd-9277-5ee1d452dd1e","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-rfrb2\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:13.237Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"ce4ab659-34bc-48fd-9277-5ee1d452dd1e"}
{"level":"error","ts":"2026-07-13T16:08:13.237Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"ce4ab659-34bc-48fd-9277-5ee1d452dd1e","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-rfrb2\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:13.237Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"68276220-6fe8-40d3-92df-148189e0aa51","namespacedName":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:13.237Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"68276220-6fe8-40d3-92df-148189e0aa51","mode":"IntegratedOAuth","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:13.237Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"68276220-6fe8-40d3-92df-148189e0aa51","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:13.321Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"719165a6-4b22-44da-8caf-c6be77db6935"}
{"level":"info","ts":"2026-07-13T16:08:13.321Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"719165a6-4b22-44da-8caf-c6be77db6935"}
{"level":"info","ts":"2026-07-13T16:08:13.321Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"719165a6-4b22-44da-8caf-c6be77db6935","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:13.321Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"719165a6-4b22-44da-8caf-c6be77db6935","error":"head certificate secret ray-head-secret-long-running-rfrb2 not found"}
{"level":"error","ts":"2026-07-13T16:08:13.321Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"719165a6-4b22-44da-8caf-c6be77db6935","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:13.321Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"719165a6-4b22-44da-8caf-c6be77db6935","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found"}
{"level":"info","ts":"2026-07-13T16:08:13.321Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"719165a6-4b22-44da-8caf-c6be77db6935","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:13.321Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"719165a6-4b22-44da-8caf-c6be77db6935","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:13.321Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"48efa98e-a936-4108-8dde-5bec28e25ce6","rayCluster":"long-running-rfrb2","podIPs":[]}
{"level":"info","ts":"2026-07-13T16:08:13.321Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"719165a6-4b22-44da-8caf-c6be77db6935","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:13.321Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"719165a6-4b22-44da-8caf-c6be77db6935"}
{"level":"error","ts":"2026-07-13T16:08:13.321Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"719165a6-4b22-44da-8caf-c6be77db6935","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:08:13.321Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"48efa98e-a936-4108-8dde-5bec28e25ce6","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-rfrb2\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:13.321Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"48efa98e-a936-4108-8dde-5bec28e25ce6"}
{"level":"error","ts":"2026-07-13T16:08:13.321Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"48efa98e-a936-4108-8dde-5bec28e25ce6","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-rfrb2\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:13.322Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"a69b3df4-aaad-4709-880b-e15cbb82233c","rayCluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:13.322Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"a69b3df4-aaad-4709-880b-e15cbb82233c"}
{"level":"info","ts":"2026-07-13T16:08:13.322Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"c56cc1a9-ae05-4b54-9229-2e89ab6758e5","name":"long-running-rfrb2-head"}
{"level":"info","ts":"2026-07-13T16:08:13.327Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"a0b8dad5-e67f-458f-a774-418ae6e0e48b","rayCluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:13.328Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"a0b8dad5-e67f-458f-a774-418ae6e0e48b"}
{"level":"info","ts":"2026-07-13T16:08:13.338Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"68276220-6fe8-40d3-92df-148189e0aa51","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:13.421Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"05b29364-09da-4835-a99b-d4634cb4c578","namespacedName":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:13.421Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"c56cc1a9-ae05-4b54-9229-2e89ab6758e5","name":"long-running-rfrb2-workers"}
{"level":"info","ts":"2026-07-13T16:08:13.421Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b23921d1-26c9-44b5-872d-d24afe1dd88d"}
{"level":"info","ts":"2026-07-13T16:08:13.421Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"c56cc1a9-ae05-4b54-9229-2e89ab6758e5","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:13.421Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b23921d1-26c9-44b5-872d-d24afe1dd88d"}
{"level":"info","ts":"2026-07-13T16:08:13.421Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"05b29364-09da-4835-a99b-d4634cb4c578","mode":"IntegratedOAuth","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:13.421Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"05b29364-09da-4835-a99b-d4634cb4c578","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:13.421Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b23921d1-26c9-44b5-872d-d24afe1dd88d","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:13.421Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b23921d1-26c9-44b5-872d-d24afe1dd88d","error":"head certificate secret ray-head-secret-long-running-rfrb2 not found"}
{"level":"error","ts":"2026-07-13T16:08:13.421Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b23921d1-26c9-44b5-872d-d24afe1dd88d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:13.421Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b23921d1-26c9-44b5-872d-d24afe1dd88d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found"}
{"level":"info","ts":"2026-07-13T16:08:13.421Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b23921d1-26c9-44b5-872d-d24afe1dd88d","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:13.421Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b23921d1-26c9-44b5-872d-d24afe1dd88d","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:13.421Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b23921d1-26c9-44b5-872d-d24afe1dd88d","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:13.421Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b23921d1-26c9-44b5-872d-d24afe1dd88d"}
{"level":"error","ts":"2026-07-13T16:08:13.421Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b23921d1-26c9-44b5-872d-d24afe1dd88d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:13.422Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"1451494d-2967-49fe-9fde-6888c8e42564","rayCluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:13.422Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"64858dd2-3764-4934-a073-509bd7cdf03e","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:13.422Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"f8707561-781d-4ee5-ac89-c8568f5ce5b1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:13.422Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"f8707561-781d-4ee5-ac89-c8568f5ce5b1","name":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:13.422Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"1451494d-2967-49fe-9fde-6888c8e42564"}
{"level":"info","ts":"2026-07-13T16:08:13.422Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"f8707561-781d-4ee5-ac89-c8568f5ce5b1","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:13.423Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"f8707561-781d-4ee5-ac89-c8568f5ce5b1"}
{"level":"info","ts":"2026-07-13T16:08:13.423Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"f8707561-781d-4ee5-ac89-c8568f5ce5b1","RayCluster":"long-running-rfrb2","State":""}
{"level":"error","ts":"2026-07-13T16:08:13.445Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"05b29364-09da-4835-a99b-d4634cb4c578","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-rfrb2\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:13.445Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"05b29364-09da-4835-a99b-d4634cb4c578"}
{"level":"error","ts":"2026-07-13T16:08:13.445Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"05b29364-09da-4835-a99b-d4634cb4c578","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-rfrb2\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:13.450Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"64858dd2-3764-4934-a073-509bd7cdf03e","name":"long-running-rfrb2-head"}
{"level":"info","ts":"2026-07-13T16:08:13.451Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"353b4087-54fd-46ef-b28b-6d2b2c66ca9a","namespacedName":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:13.451Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"353b4087-54fd-46ef-b28b-6d2b2c66ca9a","mode":"IntegratedOAuth","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:13.451Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"353b4087-54fd-46ef-b28b-6d2b2c66ca9a","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:13.472Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"353b4087-54fd-46ef-b28b-6d2b2c66ca9a","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:13.530Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"64858dd2-3764-4934-a073-509bd7cdf03e","name":"long-running-rfrb2-workers"}
{"level":"info","ts":"2026-07-13T16:08:13.530Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"64858dd2-3764-4934-a073-509bd7cdf03e","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:13.582Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1f3b3684-2122-4fe5-abfd-d374cc7cbea4"}
{"level":"info","ts":"2026-07-13T16:08:13.582Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1f3b3684-2122-4fe5-abfd-d374cc7cbea4"}
{"level":"info","ts":"2026-07-13T16:08:13.582Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1f3b3684-2122-4fe5-abfd-d374cc7cbea4","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:13.582Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1f3b3684-2122-4fe5-abfd-d374cc7cbea4","error":"head certificate secret ray-head-secret-long-running-rfrb2 not found"}
{"level":"error","ts":"2026-07-13T16:08:13.582Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1f3b3684-2122-4fe5-abfd-d374cc7cbea4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:13.582Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1f3b3684-2122-4fe5-abfd-d374cc7cbea4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found"}
{"level":"info","ts":"2026-07-13T16:08:13.582Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1f3b3684-2122-4fe5-abfd-d374cc7cbea4","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:13.582Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1f3b3684-2122-4fe5-abfd-d374cc7cbea4","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:13.582Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1f3b3684-2122-4fe5-abfd-d374cc7cbea4","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:13.583Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1f3b3684-2122-4fe5-abfd-d374cc7cbea4"}
{"level":"error","ts":"2026-07-13T16:08:13.583Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1f3b3684-2122-4fe5-abfd-d374cc7cbea4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:13.903Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8091e8f2-a717-4ed1-9d8a-c115f61e8719"}
{"level":"info","ts":"2026-07-13T16:08:13.903Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8091e8f2-a717-4ed1-9d8a-c115f61e8719"}
{"level":"info","ts":"2026-07-13T16:08:13.903Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8091e8f2-a717-4ed1-9d8a-c115f61e8719","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:13.903Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8091e8f2-a717-4ed1-9d8a-c115f61e8719","error":"head certificate secret ray-head-secret-long-running-rfrb2 not found"}
{"level":"error","ts":"2026-07-13T16:08:13.903Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8091e8f2-a717-4ed1-9d8a-c115f61e8719","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:13.903Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8091e8f2-a717-4ed1-9d8a-c115f61e8719","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found"}
{"level":"info","ts":"2026-07-13T16:08:13.903Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8091e8f2-a717-4ed1-9d8a-c115f61e8719","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:13.903Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8091e8f2-a717-4ed1-9d8a-c115f61e8719","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:13.903Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8091e8f2-a717-4ed1-9d8a-c115f61e8719","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:13.903Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8091e8f2-a717-4ed1-9d8a-c115f61e8719"}
{"level":"error","ts":"2026-07-13T16:08:13.903Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8091e8f2-a717-4ed1-9d8a-c115f61e8719","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-rfrb2 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:14.544Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8c1d632c-be6a-4be5-b873-36bdda5bf094"}
{"level":"info","ts":"2026-07-13T16:08:14.544Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8c1d632c-be6a-4be5-b873-36bdda5bf094"}
{"level":"info","ts":"2026-07-13T16:08:14.544Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8c1d632c-be6a-4be5-b873-36bdda5bf094","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:14.544Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8c1d632c-be6a-4be5-b873-36bdda5bf094","error":"worker certificate secret ray-worker-secret-long-running-rfrb2 not found"}
{"level":"error","ts":"2026-07-13T16:08:14.544Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8c1d632c-be6a-4be5-b873-36bdda5bf094","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-long-running-rfrb2 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:14.544Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8c1d632c-be6a-4be5-b873-36bdda5bf094","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-long-running-rfrb2 not found"}
{"level":"info","ts":"2026-07-13T16:08:14.544Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8c1d632c-be6a-4be5-b873-36bdda5bf094","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:14.544Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8c1d632c-be6a-4be5-b873-36bdda5bf094","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:14.544Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8c1d632c-be6a-4be5-b873-36bdda5bf094","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:14.544Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8c1d632c-be6a-4be5-b873-36bdda5bf094"}
{"level":"error","ts":"2026-07-13T16:08:14.544Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8c1d632c-be6a-4be5-b873-36bdda5bf094","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-long-running-rfrb2 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:15.825Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb"}
{"level":"info","ts":"2026-07-13T16:08:15.825Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb"}
{"level":"info","ts":"2026-07-13T16:08:15.825Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:15.825Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb"}
{"level":"info","ts":"2026-07-13T16:08:15.825Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:08:15.825Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb"}
{"level":"info","ts":"2026-07-13T16:08:15.825Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","mode":"IntegratedOAuth","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:15.825Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","cluster":"long-running-rfrb2","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:08:15.825Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","cluster":"long-running-rfrb2","authType":"IntegratedOAuth","serviceAccount":"long-running-rfrb2-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T16:08:15.825Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-rfrb2","ray.io/group":"headgroup","ray.io/identifier":"long-running-rfrb2-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:08:15.825Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T16:08:15.825Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T16:08:15.825Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T16:08:15.825Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","enabled":true}
{"level":"info","ts":"2026-07-13T16:08:15.842Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","name":"long-running-rfrb2-head-q8chr"}
{"level":"info","ts":"2026-07-13T16:08:15.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:15.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:15.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:08:15.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:15.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:08:15.842Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb"}
{"level":"info","ts":"2026-07-13T16:08:15.842Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","nodeType":"worker","rayStartParams":{"address":"long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:08:15.842Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","rayStartCmd":"ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:15.842Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:15.842Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","enabled":true}
{"level":"info","ts":"2026-07-13T16:08:15.855Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","name":"long-running-rfrb2-small-group-worker-7gn25"}
{"level":"info","ts":"2026-07-13T16:08:15.855Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:15.855Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-rfrb2-head-svc"},"newHead":{"podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"}}
{"level":"info","ts":"2026-07-13T16:08:15.855Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2673bf95-e5e2-4180-9665-7ccdeaa444bb","name":"long-running-rfrb2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:08:15.870Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"edf06e54-ff04-4ec9-96ae-c07fbb655f59"}
{"level":"info","ts":"2026-07-13T16:08:15.870Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"edf06e54-ff04-4ec9-96ae-c07fbb655f59"}
{"level":"info","ts":"2026-07-13T16:08:15.871Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"edf06e54-ff04-4ec9-96ae-c07fbb655f59","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:15.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"edf06e54-ff04-4ec9-96ae-c07fbb655f59","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:15.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"edf06e54-ff04-4ec9-96ae-c07fbb655f59","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:15.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"edf06e54-ff04-4ec9-96ae-c07fbb655f59","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:15.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"edf06e54-ff04-4ec9-96ae-c07fbb655f59","worker Pod":"long-running-rfrb2-small-group-worker-7gn25","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-rfrb2-small-group-worker-7gn25. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:15.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"edf06e54-ff04-4ec9-96ae-c07fbb655f59","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:15.871Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"91d72f5a-9362-40f9-a93e-91c54d484c09","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:15.871Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"864a0256-1ebc-4f75-97d8-bc5f617fe60c","rayCluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:15.871Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"3b894784-4c4b-4f4f-a3b1-467739eb401c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:15.871Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"3b894784-4c4b-4f4f-a3b1-467739eb401c","name":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:15.871Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"864a0256-1ebc-4f75-97d8-bc5f617fe60c","rayCluster":"long-running-rfrb2","podIPs":[]}
{"level":"info","ts":"2026-07-13T16:08:15.871Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"3b894784-4c4b-4f4f-a3b1-467739eb401c","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:15.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"edf06e54-ff04-4ec9-96ae-c07fbb655f59","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:08:15.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"edf06e54-ff04-4ec9-96ae-c07fbb655f59","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:15.871Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"3b894784-4c4b-4f4f-a3b1-467739eb401c"}
{"level":"info","ts":"2026-07-13T16:08:15.871Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"3b894784-4c4b-4f4f-a3b1-467739eb401c","RayCluster":"long-running-rfrb2","State":""}
{"level":"info","ts":"2026-07-13T16:08:15.871Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"edf06e54-ff04-4ec9-96ae-c07fbb655f59","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:15.871Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"edf06e54-ff04-4ec9-96ae-c07fbb655f59","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:15.871Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"edf06e54-ff04-4ec9-96ae-c07fbb655f59","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-rfrb2-head-svc"},"newHead":{"podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"}}
{"level":"info","ts":"2026-07-13T16:08:15.871Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"edf06e54-ff04-4ec9-96ae-c07fbb655f59","name":"long-running-rfrb2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:08:15.878Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"edf06e54-ff04-4ec9-96ae-c07fbb655f59","name":"long-running-rfrb2","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-rfrb2\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-x5qvg","name":"long-running-rfrb2"}}
{"level":"info","ts":"2026-07-13T16:08:15.878Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"edf06e54-ff04-4ec9-96ae-c07fbb655f59"}
{"level":"error","ts":"2026-07-13T16:08:15.878Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"edf06e54-ff04-4ec9-96ae-c07fbb655f59","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-rfrb2\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:15.878Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"95aff011-7e80-4e7d-bd48-1c5692d21d76"}
{"level":"info","ts":"2026-07-13T16:08:15.878Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"95aff011-7e80-4e7d-bd48-1c5692d21d76"}
{"level":"info","ts":"2026-07-13T16:08:15.879Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"95aff011-7e80-4e7d-bd48-1c5692d21d76","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:15.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"95aff011-7e80-4e7d-bd48-1c5692d21d76","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:15.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"95aff011-7e80-4e7d-bd48-1c5692d21d76","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:15.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"95aff011-7e80-4e7d-bd48-1c5692d21d76","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:15.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"95aff011-7e80-4e7d-bd48-1c5692d21d76","worker Pod":"long-running-rfrb2-small-group-worker-7gn25","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-rfrb2-small-group-worker-7gn25. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:15.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"95aff011-7e80-4e7d-bd48-1c5692d21d76","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:15.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"95aff011-7e80-4e7d-bd48-1c5692d21d76","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:08:15.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"95aff011-7e80-4e7d-bd48-1c5692d21d76","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:15.879Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"95aff011-7e80-4e7d-bd48-1c5692d21d76","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:15.879Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"95aff011-7e80-4e7d-bd48-1c5692d21d76","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:15.879Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"95aff011-7e80-4e7d-bd48-1c5692d21d76","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:08:15.879Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"95aff011-7e80-4e7d-bd48-1c5692d21d76","seconds":300}
{"level":"info","ts":"2026-07-13T16:08:15.921Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"5edc9e23-d65e-4fc1-a5e1-f3f0ad05f07d"}
{"level":"info","ts":"2026-07-13T16:08:15.921Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"5edc9e23-d65e-4fc1-a5e1-f3f0ad05f07d"}
{"level":"info","ts":"2026-07-13T16:08:15.921Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"5edc9e23-d65e-4fc1-a5e1-f3f0ad05f07d","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:15.921Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"91d72f5a-9362-40f9-a93e-91c54d484c09","name":"long-running-rfrb2-head"}
{"level":"info","ts":"2026-07-13T16:08:15.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"5edc9e23-d65e-4fc1-a5e1-f3f0ad05f07d","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:15.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"5edc9e23-d65e-4fc1-a5e1-f3f0ad05f07d","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:15.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"5edc9e23-d65e-4fc1-a5e1-f3f0ad05f07d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:15.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"5edc9e23-d65e-4fc1-a5e1-f3f0ad05f07d","worker Pod":"long-running-rfrb2-small-group-worker-7gn25","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-rfrb2-small-group-worker-7gn25. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:15.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"5edc9e23-d65e-4fc1-a5e1-f3f0ad05f07d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:15.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"5edc9e23-d65e-4fc1-a5e1-f3f0ad05f07d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:08:15.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"5edc9e23-d65e-4fc1-a5e1-f3f0ad05f07d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:15.921Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"5edc9e23-d65e-4fc1-a5e1-f3f0ad05f07d","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:15.921Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"5edc9e23-d65e-4fc1-a5e1-f3f0ad05f07d","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:15.921Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"5edc9e23-d65e-4fc1-a5e1-f3f0ad05f07d","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T16:08:15.921Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"5edc9e23-d65e-4fc1-a5e1-f3f0ad05f07d","name":"long-running-rfrb2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:08:15.930Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"a8af2c0f-574f-45ab-9b96-7053f8f41144","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:15.930Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"a8af2c0f-574f-45ab-9b96-7053f8f41144","name":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:15.930Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"a8af2c0f-574f-45ab-9b96-7053f8f41144","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:15.930Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"a8af2c0f-574f-45ab-9b96-7053f8f41144"}
{"level":"info","ts":"2026-07-13T16:08:15.930Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"a8af2c0f-574f-45ab-9b96-7053f8f41144","RayCluster":"long-running-rfrb2","State":""}
{"level":"info","ts":"2026-07-13T16:08:15.932Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fbf55a12-ea6b-48bf-ab5e-70c726250bac"}
{"level":"info","ts":"2026-07-13T16:08:15.932Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fbf55a12-ea6b-48bf-ab5e-70c726250bac"}
{"level":"info","ts":"2026-07-13T16:08:15.932Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fbf55a12-ea6b-48bf-ab5e-70c726250bac","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:15.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fbf55a12-ea6b-48bf-ab5e-70c726250bac","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:15.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fbf55a12-ea6b-48bf-ab5e-70c726250bac","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:15.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fbf55a12-ea6b-48bf-ab5e-70c726250bac","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:15.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fbf55a12-ea6b-48bf-ab5e-70c726250bac","worker Pod":"long-running-rfrb2-small-group-worker-7gn25","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-rfrb2-small-group-worker-7gn25. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:15.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fbf55a12-ea6b-48bf-ab5e-70c726250bac","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:15.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fbf55a12-ea6b-48bf-ab5e-70c726250bac","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:08:15.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fbf55a12-ea6b-48bf-ab5e-70c726250bac","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:15.932Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fbf55a12-ea6b-48bf-ab5e-70c726250bac","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:15.932Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fbf55a12-ea6b-48bf-ab5e-70c726250bac","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:15.932Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fbf55a12-ea6b-48bf-ab5e-70c726250bac","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T16:08:15.933Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fbf55a12-ea6b-48bf-ab5e-70c726250bac","name":"long-running-rfrb2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:08:15.933Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"c2955707-fdb0-4db6-9b56-5aea444420f5","rayCluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:15.933Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"6ead8541-6e62-4c63-949b-0ca1f0488b1e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:15.933Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"6ead8541-6e62-4c63-949b-0ca1f0488b1e","name":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:15.933Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"6ead8541-6e62-4c63-949b-0ca1f0488b1e","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:08:15.933Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"c2955707-fdb0-4db6-9b56-5aea444420f5","rayCluster":"long-running-rfrb2","podIPs":[]}
{"level":"info","ts":"2026-07-13T16:08:15.933Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"6ead8541-6e62-4c63-949b-0ca1f0488b1e"}
{"level":"info","ts":"2026-07-13T16:08:15.933Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"6ead8541-6e62-4c63-949b-0ca1f0488b1e","RayCluster":"long-running-rfrb2","State":""}
{"level":"info","ts":"2026-07-13T16:08:15.934Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"91d72f5a-9362-40f9-a93e-91c54d484c09","name":"long-running-rfrb2-workers"}
{"level":"info","ts":"2026-07-13T16:08:15.934Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"91d72f5a-9362-40f9-a93e-91c54d484c09","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:15.934Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"87309beb-2b87-4242-a1e5-e8ec97694f26","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:15.938Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fbf55a12-ea6b-48bf-ab5e-70c726250bac","name":"long-running-rfrb2","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-rfrb2\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-x5qvg","name":"long-running-rfrb2"}}
{"level":"info","ts":"2026-07-13T16:08:15.938Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fbf55a12-ea6b-48bf-ab5e-70c726250bac"}
{"level":"error","ts":"2026-07-13T16:08:15.938Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fbf55a12-ea6b-48bf-ab5e-70c726250bac","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-rfrb2\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:15.943Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b34ce36d-1d00-4881-b187-9e82dbc3aa4c"}
{"level":"info","ts":"2026-07-13T16:08:15.943Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b34ce36d-1d00-4881-b187-9e82dbc3aa4c"}
{"level":"info","ts":"2026-07-13T16:08:15.943Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b34ce36d-1d00-4881-b187-9e82dbc3aa4c","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:15.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b34ce36d-1d00-4881-b187-9e82dbc3aa4c","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:15.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b34ce36d-1d00-4881-b187-9e82dbc3aa4c","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:15.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b34ce36d-1d00-4881-b187-9e82dbc3aa4c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:15.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b34ce36d-1d00-4881-b187-9e82dbc3aa4c","worker Pod":"long-running-rfrb2-small-group-worker-7gn25","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-rfrb2-small-group-worker-7gn25. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:15.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b34ce36d-1d00-4881-b187-9e82dbc3aa4c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:15.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b34ce36d-1d00-4881-b187-9e82dbc3aa4c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:08:15.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b34ce36d-1d00-4881-b187-9e82dbc3aa4c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:15.943Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b34ce36d-1d00-4881-b187-9e82dbc3aa4c","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:15.943Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b34ce36d-1d00-4881-b187-9e82dbc3aa4c","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:15.943Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b34ce36d-1d00-4881-b187-9e82dbc3aa4c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:08:15.943Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b34ce36d-1d00-4881-b187-9e82dbc3aa4c","seconds":300}
{"level":"info","ts":"2026-07-13T16:08:15.943Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"87309beb-2b87-4242-a1e5-e8ec97694f26","name":"long-running-rfrb2-head"}
{"level":"info","ts":"2026-07-13T16:08:16.025Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"87309beb-2b87-4242-a1e5-e8ec97694f26","name":"long-running-rfrb2-workers"}
{"level":"info","ts":"2026-07-13T16:08:16.025Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"87309beb-2b87-4242-a1e5-e8ec97694f26","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:16.289Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ffd7a98a-f689-45e5-963c-09a505814a43"}
{"level":"info","ts":"2026-07-13T16:08:16.289Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ffd7a98a-f689-45e5-963c-09a505814a43"}
{"level":"info","ts":"2026-07-13T16:08:16.289Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ffd7a98a-f689-45e5-963c-09a505814a43","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:16.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ffd7a98a-f689-45e5-963c-09a505814a43","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:16.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ffd7a98a-f689-45e5-963c-09a505814a43","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:16.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ffd7a98a-f689-45e5-963c-09a505814a43","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:16.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ffd7a98a-f689-45e5-963c-09a505814a43","worker Pod":"long-running-rfrb2-small-group-worker-7gn25","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-rfrb2-small-group-worker-7gn25. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:16.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ffd7a98a-f689-45e5-963c-09a505814a43","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:16.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ffd7a98a-f689-45e5-963c-09a505814a43","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:08:16.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ffd7a98a-f689-45e5-963c-09a505814a43","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:16.289Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ffd7a98a-f689-45e5-963c-09a505814a43","podName":"long-running-rfrb2-small-group-worker-7gn25","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:16.290Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ffd7a98a-f689-45e5-963c-09a505814a43","podName":"long-running-rfrb2-small-group-worker-7gn25","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:16.290Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ffd7a98a-f689-45e5-963c-09a505814a43","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:08:16.290Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ffd7a98a-f689-45e5-963c-09a505814a43","seconds":300}
{"level":"info","ts":"2026-07-13T16:08:16.310Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"252145f5-cee8-48fa-9a62-041a76833325"}
{"level":"info","ts":"2026-07-13T16:08:16.310Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"252145f5-cee8-48fa-9a62-041a76833325"}
{"level":"info","ts":"2026-07-13T16:08:16.310Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"252145f5-cee8-48fa-9a62-041a76833325","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:16.310Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"252145f5-cee8-48fa-9a62-041a76833325","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:16.310Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"252145f5-cee8-48fa-9a62-041a76833325","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:16.310Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"252145f5-cee8-48fa-9a62-041a76833325","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:16.310Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"252145f5-cee8-48fa-9a62-041a76833325","worker Pod":"long-running-rfrb2-small-group-worker-7gn25","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-rfrb2-small-group-worker-7gn25. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:16.310Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"252145f5-cee8-48fa-9a62-041a76833325","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:16.310Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"252145f5-cee8-48fa-9a62-041a76833325","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:08:16.310Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"252145f5-cee8-48fa-9a62-041a76833325","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:16.310Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"252145f5-cee8-48fa-9a62-041a76833325","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:16.310Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"252145f5-cee8-48fa-9a62-041a76833325","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:16.310Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"252145f5-cee8-48fa-9a62-041a76833325","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:08:16.310Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"252145f5-cee8-48fa-9a62-041a76833325","seconds":300}
{"level":"info","ts":"2026-07-13T16:08:16.501Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"e068bef8-ec1b-4349-a099-7aac0d54dfba"}
{"level":"info","ts":"2026-07-13T16:08:16.501Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"e068bef8-ec1b-4349-a099-7aac0d54dfba"}
{"level":"info","ts":"2026-07-13T16:08:16.501Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"e068bef8-ec1b-4349-a099-7aac0d54dfba","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:16.501Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"e068bef8-ec1b-4349-a099-7aac0d54dfba","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:16.501Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"e068bef8-ec1b-4349-a099-7aac0d54dfba","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:16.501Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"e068bef8-ec1b-4349-a099-7aac0d54dfba","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:16.501Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"e068bef8-ec1b-4349-a099-7aac0d54dfba","worker Pod":"long-running-rfrb2-small-group-worker-7gn25","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-rfrb2-small-group-worker-7gn25. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:16.501Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"e068bef8-ec1b-4349-a099-7aac0d54dfba","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:16.501Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"e068bef8-ec1b-4349-a099-7aac0d54dfba","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:08:16.501Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"e068bef8-ec1b-4349-a099-7aac0d54dfba","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:16.501Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"e068bef8-ec1b-4349-a099-7aac0d54dfba","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:16.501Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"e068bef8-ec1b-4349-a099-7aac0d54dfba","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:16.501Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"e068bef8-ec1b-4349-a099-7aac0d54dfba","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:08:16.501Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"e068bef8-ec1b-4349-a099-7aac0d54dfba","seconds":300}
{"level":"info","ts":"2026-07-13T16:08:17.130Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"07c707f7-8511-4d54-8388-cf895ad81f46"}
{"level":"info","ts":"2026-07-13T16:08:17.130Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"07c707f7-8511-4d54-8388-cf895ad81f46"}
{"level":"info","ts":"2026-07-13T16:08:17.130Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"07c707f7-8511-4d54-8388-cf895ad81f46","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:17.130Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"07c707f7-8511-4d54-8388-cf895ad81f46","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:17.130Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"07c707f7-8511-4d54-8388-cf895ad81f46","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:17.130Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"07c707f7-8511-4d54-8388-cf895ad81f46","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:17.130Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"07c707f7-8511-4d54-8388-cf895ad81f46","worker Pod":"long-running-rfrb2-small-group-worker-7gn25","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-rfrb2-small-group-worker-7gn25. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:17.130Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"07c707f7-8511-4d54-8388-cf895ad81f46","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:17.130Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"07c707f7-8511-4d54-8388-cf895ad81f46","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:08:17.130Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"07c707f7-8511-4d54-8388-cf895ad81f46","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:17.130Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"07c707f7-8511-4d54-8388-cf895ad81f46","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:17.130Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"07c707f7-8511-4d54-8388-cf895ad81f46","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:17.130Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"07c707f7-8511-4d54-8388-cf895ad81f46","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"},"newHead":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"}}
{"level":"info","ts":"2026-07-13T16:08:17.130Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"07c707f7-8511-4d54-8388-cf895ad81f46","name":"long-running-rfrb2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:08:17.142Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"6fa45f45-e3c7-4033-b8af-9f94b50e0dae","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:17.142Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"40d625c9-c2a2-409f-9e8a-b979ce1aa1ba","rayCluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:17.142Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"fd67bd4b-a63d-4ff1-9b82-246c9a9bac20","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:17.142Z","logger":"controllers.RayJob","msg":"The RayJob has passed the activeDeadlineSeconds. Transition the status to `Failed`.","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"fd67bd4b-a63d-4ff1-9b82-246c9a9bac20","StartTime":"2026-07-13 16:08:12 +0000 UTC","ActiveDeadlineSeconds":5}
{"level":"info","ts":"2026-07-13T16:08:17.142Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"fd67bd4b-a63d-4ff1-9b82-246c9a9bac20","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-sfgvz","rayClusterName":"long-running-rfrb2","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T16:08:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-sfgvz","rayClusterName":"long-running-rfrb2","jobDeploymentStatus":"Failed","reason":"DeadlineExceeded","message":"The RayJob has passed the activeDeadlineSeconds. StartTime: 2026-07-13 16:08:12 +0000 UTC. ActiveDeadlineSeconds: 5","startTime":"2026-07-13T16:08:12Z","succeeded":0,"failed":1,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T16:08:17.142Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"fd67bd4b-a63d-4ff1-9b82-246c9a9bac20","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-07-13T16:08:17.142Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"40d625c9-c2a2-409f-9e8a-b979ce1aa1ba","pod":"long-running-rfrb2-small-group-worker-7gn25","ip":"10.134.0.35"}
{"level":"info","ts":"2026-07-13T16:08:17.142Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"40d625c9-c2a2-409f-9e8a-b979ce1aa1ba","pod":"long-running-rfrb2-head-q8chr","ip":"10.133.0.38"}
{"level":"info","ts":"2026-07-13T16:08:17.158Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"40d625c9-c2a2-409f-9e8a-b979ce1aa1ba","certificate":"ray-head-cert-long-running-rfrb2","dnsNames":["long-running-rfrb2-head-svc","localhost","long-running-rfrb2-head-svc.test-ns-x5qvg.svc","long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local"],"ipAddresses":["10.134.0.35","10.133.0.38","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:08:17.158Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"b8958269-8bc0-4801-a2b2-c288c096012f","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:17.158Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"b8958269-8bc0-4801-a2b2-c288c096012f","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-07-13 16:08:17 +0000 UTC","Now":"2026-07-13T16:08:17.158Z","ShutdownTime":"2026-07-13T16:18:17.000Z"}
{"level":"info","ts":"2026-07-13T16:08:17.158Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"b8958269-8bc0-4801-a2b2-c288c096012f","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T16:08:17.158Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"b8958269-8bc0-4801-a2b2-c288c096012f","seconds":601}
{"level":"info","ts":"2026-07-13T16:08:17.160Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"6fa45f45-e3c7-4033-b8af-9f94b50e0dae","name":"long-running-rfrb2-head"}
{"level":"info","ts":"2026-07-13T16:08:17.170Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"40d625c9-c2a2-409f-9e8a-b979ce1aa1ba","certificate":"ray-worker-cert-long-running-rfrb2","dnsNames":["long-running-rfrb2-worker-svc","localhost","long-running-rfrb2-small-group","long-running-rfrb2-worker-svc.test-ns-x5qvg.svc","long-running-rfrb2-worker-svc.test-ns-x5qvg.svc.cluster.local","*.long-running-rfrb2-worker-svc.test-ns-x5qvg.svc","*.long-running-rfrb2-worker-svc.test-ns-x5qvg.svc.cluster.local","*-worker-*.test-ns-x5qvg.svc","*-worker-*.test-ns-x5qvg.svc.cluster.local"],"ipAddresses":["10.134.0.35","10.133.0.38","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:08:17.170Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"40d625c9-c2a2-409f-9e8a-b979ce1aa1ba","rayCluster":"long-running-rfrb2","podIPs":["10.134.0.35","10.133.0.38"]}
{"level":"info","ts":"2026-07-13T16:08:17.173Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"6fa45f45-e3c7-4033-b8af-9f94b50e0dae","name":"long-running-rfrb2-workers"}
{"level":"info","ts":"2026-07-13T16:08:17.173Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"6fa45f45-e3c7-4033-b8af-9f94b50e0dae","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:17.961Z","logger":"controllers.RayJob","msg":"Skipping RayJob managed by a custom controller","RayJob":{"name":"managed-externally","namespace":"test-ns-x5qvg"},"reconcileID":"6f94acf5-1b5a-417b-866b-4dc6bb3f9eaf","managed-by":"kueue.x-k8s.io/multikueue"}
{"level":"info","ts":"2026-07-13T16:08:17.992Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"managed-externally","namespace":"test-ns-x5qvg"},"reconcileID":"7c672968-bc2c-4f5c-a00f-ff912e492a50"}
{"level":"info","ts":"2026-07-13T16:08:18.085Z","logger":"controllers.RayJob","msg":"Skipping RayJob managed by a custom controller","RayJob":{"name":"managed-externally","namespace":"test-ns-qbjjb"},"reconcileID":"c110a2b4-d311-4b1d-b157-e5a3bc055350","managed-by":"kueue.x-k8s.io/multikueue"}
{"level":"error","ts":"2026-07-13T16:08:18.343Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"70bc57e6-276e-4042-b033-349c575e8d3a","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:18.343Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"70bc57e6-276e-4042-b033-349c575e8d3a"}
{"level":"error","ts":"2026-07-13T16:08:18.343Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"70bc57e6-276e-4042-b033-349c575e8d3a","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:18.931Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"f7cb777d-b9c1-4537-a342-11ee857ee258","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:18.931Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"f7cb777d-b9c1-4537-a342-11ee857ee258","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-07-13 16:08:17 +0000 UTC","Now":"2026-07-13T16:08:18.931Z","ShutdownTime":"2026-07-13T16:18:17.000Z"}
{"level":"info","ts":"2026-07-13T16:08:18.931Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"f7cb777d-b9c1-4537-a342-11ee857ee258","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T16:08:18.931Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"f7cb777d-b9c1-4537-a342-11ee857ee258","seconds":600}
{"level":"info","ts":"2026-07-13T16:08:19.143Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"58467fe3-8960-4ab0-a6dd-b4f5b1ff1d2e"}
{"level":"info","ts":"2026-07-13T16:08:19.143Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"58467fe3-8960-4ab0-a6dd-b4f5b1ff1d2e"}
{"level":"info","ts":"2026-07-13T16:08:19.143Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"58467fe3-8960-4ab0-a6dd-b4f5b1ff1d2e","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:19.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"58467fe3-8960-4ab0-a6dd-b4f5b1ff1d2e","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:19.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"58467fe3-8960-4ab0-a6dd-b4f5b1ff1d2e","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:19.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"58467fe3-8960-4ab0-a6dd-b4f5b1ff1d2e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:19.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"58467fe3-8960-4ab0-a6dd-b4f5b1ff1d2e","worker Pod":"long-running-rfrb2-small-group-worker-7gn25","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-rfrb2-small-group-worker-7gn25. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:19.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"58467fe3-8960-4ab0-a6dd-b4f5b1ff1d2e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:19.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"58467fe3-8960-4ab0-a6dd-b4f5b1ff1d2e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:08:19.143Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"58467fe3-8960-4ab0-a6dd-b4f5b1ff1d2e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:19.143Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"58467fe3-8960-4ab0-a6dd-b4f5b1ff1d2e","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:19.143Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"58467fe3-8960-4ab0-a6dd-b4f5b1ff1d2e","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:19.143Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"58467fe3-8960-4ab0-a6dd-b4f5b1ff1d2e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:08:19.143Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"58467fe3-8960-4ab0-a6dd-b4f5b1ff1d2e","seconds":300}
{"level":"info","ts":"2026-07-13T16:08:21.099Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"1d8cd34b-8b30-4b86-af4e-741c5c57f3af","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:21.099Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"1d8cd34b-8b30-4b86-af4e-741c5c57f3af","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T16:08:21.107Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"1d8cd34b-8b30-4b86-af4e-741c5c57f3af"}
{"level":"info","ts":"2026-07-13T16:08:21.107Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"1d8cd34b-8b30-4b86-af4e-741c5c57f3af","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-13T16:08:21.107Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"1d8cd34b-8b30-4b86-af4e-741c5c57f3af","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-hd5d2","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T16:08:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T16:08:21.107Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"1d8cd34b-8b30-4b86-af4e-741c5c57f3af","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T16:08:21.114Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"742bbfab-3e7d-451e-9389-df27a3eef86d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:21.114Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"742bbfab-3e7d-451e-9389-df27a3eef86d","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:21.114Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"742bbfab-3e7d-451e-9389-df27a3eef86d","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:08:21.114Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"742bbfab-3e7d-451e-9389-df27a3eef86d","head service name":"raycluster-head-svc","namespace":"test-ns-qbjjb"}
{"level":"info","ts":"2026-07-13T16:08:21.114Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"742bbfab-3e7d-451e-9389-df27a3eef86d","head service URL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-13T16:08:21.114Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"742bbfab-3e7d-451e-9389-df27a3eef86d"}
{"level":"info","ts":"2026-07-13T16:08:21.114Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"742bbfab-3e7d-451e-9389-df27a3eef86d","command":["if","!","ray","job","status","--address","http://raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","counter-hd5d2",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","--no-wait","--runtime-env-json","\"{\\\"env_vars\\\":{\\\"counter_name\\\":\\\"test_counter\\\"}}\"","--submission-id","counter-hd5d2","--","python /home/ray/jobs/counter.py",";","fi",";","ray","job","logs","--address","http://raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","--follow","counter-hd5d2"]}
{"level":"info","ts":"2026-07-13T16:08:21.122Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"742bbfab-3e7d-451e-9389-df27a3eef86d","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-07-13T16:08:21.122Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"742bbfab-3e7d-451e-9389-df27a3eef86d","SubmissionMode":"K8sJobMode","RayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:21.122Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"742bbfab-3e7d-451e-9389-df27a3eef86d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-hd5d2","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T16:08:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-hd5d2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T16:08:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T16:08:21.122Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"742bbfab-3e7d-451e-9389-df27a3eef86d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T16:08:21.130Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"72f0de80-bf7d-43ae-9864-e84ae86ec354","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:21.130Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"72f0de80-bf7d-43ae-9864-e84ae86ec354","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:21.130Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"72f0de80-bf7d-43ae-9864-e84ae86ec354","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:21.142Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"72f0de80-bf7d-43ae-9864-e84ae86ec354","JobId":"counter-hd5d2","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:21.142Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"72f0de80-bf7d-43ae-9864-e84ae86ec354"}
{"level":"error","ts":"2026-07-13T16:08:21.142Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"72f0de80-bf7d-43ae-9864-e84ae86ec354","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:21.142Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"4228ca40-e4bc-475f-a960-ade05612120a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:21.142Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"4228ca40-e4bc-475f-a960-ade05612120a","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:21.142Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"4228ca40-e4bc-475f-a960-ade05612120a","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:21.147Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"4228ca40-e4bc-475f-a960-ade05612120a","JobId":"counter-hd5d2","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:21.147Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"4228ca40-e4bc-475f-a960-ade05612120a"}
{"level":"error","ts":"2026-07-13T16:08:21.147Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"4228ca40-e4bc-475f-a960-ade05612120a","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:21.148Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"0dbfb2ed-d7ff-4cd4-968c-ddb1efb1ee15","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:21.148Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"0dbfb2ed-d7ff-4cd4-968c-ddb1efb1ee15","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:21.148Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"0dbfb2ed-d7ff-4cd4-968c-ddb1efb1ee15","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:21.154Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"0dbfb2ed-d7ff-4cd4-968c-ddb1efb1ee15","JobId":"counter-hd5d2","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:21.154Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"0dbfb2ed-d7ff-4cd4-968c-ddb1efb1ee15"}
{"level":"error","ts":"2026-07-13T16:08:21.154Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"0dbfb2ed-d7ff-4cd4-968c-ddb1efb1ee15","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:21.175Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"9e2b4ca1-ad09-48c4-ab4b-890e7eb3f5da","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:21.175Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"9e2b4ca1-ad09-48c4-ab4b-890e7eb3f5da","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:21.175Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"9e2b4ca1-ad09-48c4-ab4b-890e7eb3f5da","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:21.180Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"9e2b4ca1-ad09-48c4-ab4b-890e7eb3f5da","JobId":"counter-hd5d2","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:21.180Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"9e2b4ca1-ad09-48c4-ab4b-890e7eb3f5da"}
{"level":"error","ts":"2026-07-13T16:08:21.180Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"9e2b4ca1-ad09-48c4-ab4b-890e7eb3f5da","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:21.220Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"1b1bb5d1-0a0e-4968-aea0-a025755cd06a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:21.220Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"1b1bb5d1-0a0e-4968-aea0-a025755cd06a","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:21.220Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"1b1bb5d1-0a0e-4968-aea0-a025755cd06a","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:21.225Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"1b1bb5d1-0a0e-4968-aea0-a025755cd06a","JobId":"counter-hd5d2","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:21.225Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"1b1bb5d1-0a0e-4968-aea0-a025755cd06a"}
{"level":"error","ts":"2026-07-13T16:08:21.225Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"1b1bb5d1-0a0e-4968-aea0-a025755cd06a","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:21.305Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"b53b4841-2277-4fbc-8d95-8d22a973e8a8","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:21.305Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"b53b4841-2277-4fbc-8d95-8d22a973e8a8","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:21.305Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"b53b4841-2277-4fbc-8d95-8d22a973e8a8","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:21.310Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"b53b4841-2277-4fbc-8d95-8d22a973e8a8","JobId":"counter-hd5d2","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:21.310Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"b53b4841-2277-4fbc-8d95-8d22a973e8a8"}
{"level":"error","ts":"2026-07-13T16:08:21.310Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"b53b4841-2277-4fbc-8d95-8d22a973e8a8","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:21.470Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"de5120c4-72df-4caf-895b-f7f5e8506fbc","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:21.470Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"de5120c4-72df-4caf-895b-f7f5e8506fbc","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:21.470Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"de5120c4-72df-4caf-895b-f7f5e8506fbc","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:21.476Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"de5120c4-72df-4caf-895b-f7f5e8506fbc","JobId":"counter-hd5d2","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:21.476Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"de5120c4-72df-4caf-895b-f7f5e8506fbc"}
{"level":"error","ts":"2026-07-13T16:08:21.476Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"de5120c4-72df-4caf-895b-f7f5e8506fbc","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:21.522Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2e4c4c0b-9efb-4cbb-abf3-5de700ebd9ba"}
{"level":"info","ts":"2026-07-13T16:08:21.522Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2e4c4c0b-9efb-4cbb-abf3-5de700ebd9ba"}
{"level":"info","ts":"2026-07-13T16:08:21.522Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2e4c4c0b-9efb-4cbb-abf3-5de700ebd9ba","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:21.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2e4c4c0b-9efb-4cbb-abf3-5de700ebd9ba","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:21.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2e4c4c0b-9efb-4cbb-abf3-5de700ebd9ba","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:21.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2e4c4c0b-9efb-4cbb-abf3-5de700ebd9ba","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:21.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2e4c4c0b-9efb-4cbb-abf3-5de700ebd9ba","worker Pod":"long-running-rfrb2-small-group-worker-7gn25","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-rfrb2-small-group-worker-7gn25. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:21.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2e4c4c0b-9efb-4cbb-abf3-5de700ebd9ba","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:21.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2e4c4c0b-9efb-4cbb-abf3-5de700ebd9ba","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:08:21.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2e4c4c0b-9efb-4cbb-abf3-5de700ebd9ba","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:21.522Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2e4c4c0b-9efb-4cbb-abf3-5de700ebd9ba","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:21.522Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2e4c4c0b-9efb-4cbb-abf3-5de700ebd9ba","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:21.522Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2e4c4c0b-9efb-4cbb-abf3-5de700ebd9ba","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:08:21.522Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"2e4c4c0b-9efb-4cbb-abf3-5de700ebd9ba","seconds":300}
{"level":"info","ts":"2026-07-13T16:08:21.797Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"56e6f942-f446-41b7-853f-5ae2ecff67c5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:21.797Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"56e6f942-f446-41b7-853f-5ae2ecff67c5","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:21.797Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"56e6f942-f446-41b7-853f-5ae2ecff67c5","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:21.802Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"56e6f942-f446-41b7-853f-5ae2ecff67c5","JobId":"counter-hd5d2","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:21.803Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"56e6f942-f446-41b7-853f-5ae2ecff67c5"}
{"level":"error","ts":"2026-07-13T16:08:21.803Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"56e6f942-f446-41b7-853f-5ae2ecff67c5","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:22.147Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d79c1f7c-0418-42ac-b22d-103d29675521"}
{"level":"info","ts":"2026-07-13T16:08:22.147Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d79c1f7c-0418-42ac-b22d-103d29675521"}
{"level":"info","ts":"2026-07-13T16:08:22.147Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d79c1f7c-0418-42ac-b22d-103d29675521","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:22.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d79c1f7c-0418-42ac-b22d-103d29675521","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:22.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d79c1f7c-0418-42ac-b22d-103d29675521","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:22.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d79c1f7c-0418-42ac-b22d-103d29675521","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:22.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d79c1f7c-0418-42ac-b22d-103d29675521","worker Pod":"long-running-rfrb2-small-group-worker-7gn25","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-rfrb2-small-group-worker-7gn25. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:22.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d79c1f7c-0418-42ac-b22d-103d29675521","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:22.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d79c1f7c-0418-42ac-b22d-103d29675521","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:08:22.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d79c1f7c-0418-42ac-b22d-103d29675521","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:22.147Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d79c1f7c-0418-42ac-b22d-103d29675521","podName":"long-running-rfrb2-small-group-worker-7gn25","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:22.147Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d79c1f7c-0418-42ac-b22d-103d29675521","podName":"long-running-rfrb2-small-group-worker-7gn25","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:22.147Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d79c1f7c-0418-42ac-b22d-103d29675521","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:08:22.147Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d79c1f7c-0418-42ac-b22d-103d29675521","seconds":300}
{"level":"info","ts":"2026-07-13T16:08:22.443Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"a09dd742-68d4-402f-a380-07d9346a2ded","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:22.443Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"a09dd742-68d4-402f-a380-07d9346a2ded","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:22.443Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"a09dd742-68d4-402f-a380-07d9346a2ded","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:22.449Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"a09dd742-68d4-402f-a380-07d9346a2ded","JobId":"counter-hd5d2","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:22.449Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"a09dd742-68d4-402f-a380-07d9346a2ded"}
{"level":"error","ts":"2026-07-13T16:08:22.449Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"a09dd742-68d4-402f-a380-07d9346a2ded","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:22.530Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"60f618b3-a9ce-4d0e-971d-d0f456e9ccb3"}
{"level":"info","ts":"2026-07-13T16:08:22.530Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"60f618b3-a9ce-4d0e-971d-d0f456e9ccb3"}
{"level":"info","ts":"2026-07-13T16:08:22.530Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"60f618b3-a9ce-4d0e-971d-d0f456e9ccb3","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:22.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"60f618b3-a9ce-4d0e-971d-d0f456e9ccb3","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:22.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"60f618b3-a9ce-4d0e-971d-d0f456e9ccb3","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:22.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"60f618b3-a9ce-4d0e-971d-d0f456e9ccb3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:22.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"60f618b3-a9ce-4d0e-971d-d0f456e9ccb3","worker Pod":"long-running-rfrb2-small-group-worker-7gn25","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-rfrb2-small-group-worker-7gn25. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:22.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"60f618b3-a9ce-4d0e-971d-d0f456e9ccb3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:22.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"60f618b3-a9ce-4d0e-971d-d0f456e9ccb3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:08:22.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"60f618b3-a9ce-4d0e-971d-d0f456e9ccb3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:22.530Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"60f618b3-a9ce-4d0e-971d-d0f456e9ccb3","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:22.530Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"60f618b3-a9ce-4d0e-971d-d0f456e9ccb3","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:22.530Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"60f618b3-a9ce-4d0e-971d-d0f456e9ccb3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:08:22.530Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"60f618b3-a9ce-4d0e-971d-d0f456e9ccb3","seconds":300}
{"level":"info","ts":"2026-07-13T16:08:23.096Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fe74a485-dcd3-4f72-9b9c-f1c4a8080023"}
{"level":"info","ts":"2026-07-13T16:08:23.096Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fe74a485-dcd3-4f72-9b9c-f1c4a8080023"}
{"level":"info","ts":"2026-07-13T16:08:23.097Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fe74a485-dcd3-4f72-9b9c-f1c4a8080023","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:23.097Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fe74a485-dcd3-4f72-9b9c-f1c4a8080023","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:23.097Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fe74a485-dcd3-4f72-9b9c-f1c4a8080023","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:23.097Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fe74a485-dcd3-4f72-9b9c-f1c4a8080023","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:23.097Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fe74a485-dcd3-4f72-9b9c-f1c4a8080023","worker Pod":"long-running-rfrb2-small-group-worker-7gn25","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-rfrb2-small-group-worker-7gn25. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:23.097Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fe74a485-dcd3-4f72-9b9c-f1c4a8080023","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:23.097Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fe74a485-dcd3-4f72-9b9c-f1c4a8080023","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:08:23.097Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fe74a485-dcd3-4f72-9b9c-f1c4a8080023","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:23.097Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fe74a485-dcd3-4f72-9b9c-f1c4a8080023","podName":"long-running-rfrb2-small-group-worker-7gn25","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:23.097Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fe74a485-dcd3-4f72-9b9c-f1c4a8080023","podName":"long-running-rfrb2-small-group-worker-7gn25","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:23.097Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fe74a485-dcd3-4f72-9b9c-f1c4a8080023","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:08:23.097Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"fe74a485-dcd3-4f72-9b9c-f1c4a8080023","seconds":300}
{"level":"info","ts":"2026-07-13T16:08:23.102Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"f3b9f229-c8d2-473d-924b-fe881d201631"}
{"level":"info","ts":"2026-07-13T16:08:23.102Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"f3b9f229-c8d2-473d-924b-fe881d201631"}
{"level":"info","ts":"2026-07-13T16:08:23.102Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"f3b9f229-c8d2-473d-924b-fe881d201631","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:23.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"f3b9f229-c8d2-473d-924b-fe881d201631","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:23.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"f3b9f229-c8d2-473d-924b-fe881d201631","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:23.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"f3b9f229-c8d2-473d-924b-fe881d201631","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:23.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"f3b9f229-c8d2-473d-924b-fe881d201631","worker Pod":"long-running-rfrb2-small-group-worker-7gn25","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-rfrb2-small-group-worker-7gn25. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:23.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"f3b9f229-c8d2-473d-924b-fe881d201631","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:23.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"f3b9f229-c8d2-473d-924b-fe881d201631","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:08:23.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"f3b9f229-c8d2-473d-924b-fe881d201631","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:23.102Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"f3b9f229-c8d2-473d-924b-fe881d201631","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:23.102Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"f3b9f229-c8d2-473d-924b-fe881d201631","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:08:23.102Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"f3b9f229-c8d2-473d-924b-fe881d201631","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:08:23.102Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"f3b9f229-c8d2-473d-924b-fe881d201631","seconds":300}
{"level":"info","ts":"2026-07-13T16:08:23.154Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a6da00a4-be3a-4dad-84ee-2b72e5c8da63"}
{"level":"info","ts":"2026-07-13T16:08:23.155Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a6da00a4-be3a-4dad-84ee-2b72e5c8da63"}
{"level":"info","ts":"2026-07-13T16:08:23.155Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a6da00a4-be3a-4dad-84ee-2b72e5c8da63","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:23.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a6da00a4-be3a-4dad-84ee-2b72e5c8da63","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:23.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a6da00a4-be3a-4dad-84ee-2b72e5c8da63","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:23.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a6da00a4-be3a-4dad-84ee-2b72e5c8da63","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:23.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a6da00a4-be3a-4dad-84ee-2b72e5c8da63","worker Pod":"long-running-rfrb2-small-group-worker-7gn25","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-rfrb2-small-group-worker-7gn25. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:23.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a6da00a4-be3a-4dad-84ee-2b72e5c8da63","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:23.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a6da00a4-be3a-4dad-84ee-2b72e5c8da63","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:08:23.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a6da00a4-be3a-4dad-84ee-2b72e5c8da63","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:23.155Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a6da00a4-be3a-4dad-84ee-2b72e5c8da63","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:23.155Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a6da00a4-be3a-4dad-84ee-2b72e5c8da63","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:23.155Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a6da00a4-be3a-4dad-84ee-2b72e5c8da63","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T16:08:23.155Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a6da00a4-be3a-4dad-84ee-2b72e5c8da63","name":"long-running-rfrb2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:08:23.172Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"66bf810e-a8de-4c3c-8351-81a4fe25a9a7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:23.172Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"66bf810e-a8de-4c3c-8351-81a4fe25a9a7","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:23.172Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"66bf810e-a8de-4c3c-8351-81a4fe25a9a7","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:08:23.172Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"7f2cca7f-29f2-44d2-a8a1-0f1ea55d3c87","rayCluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:23.172Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"c446f196-fc66-4174-a87a-819ef58fddef","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:23.173Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"7f2cca7f-29f2-44d2-a8a1-0f1ea55d3c87","pod":"long-running-rfrb2-head-q8chr","ip":"10.133.0.38"}
{"level":"info","ts":"2026-07-13T16:08:23.173Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"7f2cca7f-29f2-44d2-a8a1-0f1ea55d3c87","pod":"long-running-rfrb2-small-group-worker-7gn25","ip":"10.134.0.35"}
{"level":"info","ts":"2026-07-13T16:08:23.173Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"7f2cca7f-29f2-44d2-a8a1-0f1ea55d3c87","rayCluster":"long-running-rfrb2","podIPs":["10.133.0.38","10.134.0.35"]}
{"level":"error","ts":"2026-07-13T16:08:23.178Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"66bf810e-a8de-4c3c-8351-81a4fe25a9a7","JobId":"counter-hd5d2","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:23.178Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"66bf810e-a8de-4c3c-8351-81a4fe25a9a7"}
{"level":"error","ts":"2026-07-13T16:08:23.178Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"66bf810e-a8de-4c3c-8351-81a4fe25a9a7","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:23.178Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"2689a666-dbdf-48d1-a200-0b2b5112d89d","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:23.178Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"2689a666-dbdf-48d1-a200-0b2b5112d89d","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-07-13 16:08:17 +0000 UTC","Now":"2026-07-13T16:08:23.178Z","ShutdownTime":"2026-07-13T16:18:17.000Z"}
{"level":"info","ts":"2026-07-13T16:08:23.178Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"2689a666-dbdf-48d1-a200-0b2b5112d89d","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T16:08:23.178Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"2689a666-dbdf-48d1-a200-0b2b5112d89d","seconds":595}
{"level":"info","ts":"2026-07-13T16:08:23.179Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"c446f196-fc66-4174-a87a-819ef58fddef","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:23.533Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7f694190-89ff-4e17-b66a-f6c83cf55123"}
{"level":"info","ts":"2026-07-13T16:08:23.534Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7f694190-89ff-4e17-b66a-f6c83cf55123"}
{"level":"info","ts":"2026-07-13T16:08:23.534Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7f694190-89ff-4e17-b66a-f6c83cf55123","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:23.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7f694190-89ff-4e17-b66a-f6c83cf55123","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:23.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7f694190-89ff-4e17-b66a-f6c83cf55123","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:23.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7f694190-89ff-4e17-b66a-f6c83cf55123","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:23.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7f694190-89ff-4e17-b66a-f6c83cf55123","worker Pod":"long-running-rfrb2-small-group-worker-7gn25","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-rfrb2-small-group-worker-7gn25. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:23.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7f694190-89ff-4e17-b66a-f6c83cf55123","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:23.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7f694190-89ff-4e17-b66a-f6c83cf55123","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:08:23.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7f694190-89ff-4e17-b66a-f6c83cf55123","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:23.534Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7f694190-89ff-4e17-b66a-f6c83cf55123","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:23.534Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7f694190-89ff-4e17-b66a-f6c83cf55123","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:23.534Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7f694190-89ff-4e17-b66a-f6c83cf55123","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:08:23.534Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7f694190-89ff-4e17-b66a-f6c83cf55123","seconds":300}
{"level":"info","ts":"2026-07-13T16:08:23.730Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"a1739946-2f35-45be-ac46-9c8bc88ba9aa","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:23.730Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"a1739946-2f35-45be-ac46-9c8bc88ba9aa","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:23.730Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"a1739946-2f35-45be-ac46-9c8bc88ba9aa","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:23.736Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"a1739946-2f35-45be-ac46-9c8bc88ba9aa","JobId":"counter-hd5d2","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:23.736Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"a1739946-2f35-45be-ac46-9c8bc88ba9aa"}
{"level":"error","ts":"2026-07-13T16:08:23.736Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"a1739946-2f35-45be-ac46-9c8bc88ba9aa","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:25.174Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d42baa9b-75b2-4361-8629-1c8a3bb5e466"}
{"level":"info","ts":"2026-07-13T16:08:25.174Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d42baa9b-75b2-4361-8629-1c8a3bb5e466"}
{"level":"info","ts":"2026-07-13T16:08:25.174Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d42baa9b-75b2-4361-8629-1c8a3bb5e466","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:25.174Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d42baa9b-75b2-4361-8629-1c8a3bb5e466","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:25.174Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d42baa9b-75b2-4361-8629-1c8a3bb5e466","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:25.174Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d42baa9b-75b2-4361-8629-1c8a3bb5e466","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:25.175Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d42baa9b-75b2-4361-8629-1c8a3bb5e466","worker Pod":"long-running-rfrb2-small-group-worker-7gn25","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-rfrb2-small-group-worker-7gn25. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:25.175Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d42baa9b-75b2-4361-8629-1c8a3bb5e466","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:25.175Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d42baa9b-75b2-4361-8629-1c8a3bb5e466","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:08:25.175Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d42baa9b-75b2-4361-8629-1c8a3bb5e466","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:25.175Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d42baa9b-75b2-4361-8629-1c8a3bb5e466","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:25.175Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d42baa9b-75b2-4361-8629-1c8a3bb5e466","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:25.175Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d42baa9b-75b2-4361-8629-1c8a3bb5e466","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:08:25.175Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d42baa9b-75b2-4361-8629-1c8a3bb5e466","seconds":300}
{"level":"error","ts":"2026-07-13T16:08:28.583Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"70f03629-4da8-42bd-a7b0-0b909c444672","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:28.583Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"70f03629-4da8-42bd-a7b0-0b909c444672"}
{"level":"error","ts":"2026-07-13T16:08:28.583Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"70f03629-4da8-42bd-a7b0-0b909c444672","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:28.856Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"79d8004c-7cd4-40ce-9dc2-c31a7900d00f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:28.856Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"79d8004c-7cd4-40ce-9dc2-c31a7900d00f","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:28.856Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"79d8004c-7cd4-40ce-9dc2-c31a7900d00f","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:28.862Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"79d8004c-7cd4-40ce-9dc2-c31a7900d00f","JobId":"counter-hd5d2","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:28.862Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"79d8004c-7cd4-40ce-9dc2-c31a7900d00f"}
{"level":"error","ts":"2026-07-13T16:08:28.862Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"79d8004c-7cd4-40ce-9dc2-c31a7900d00f","error":"Job counter-hd5d2 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:32.280Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ab74ba99-7da5-4bd8-857a-eeaff76d2d82"}
{"level":"info","ts":"2026-07-13T16:08:32.280Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ab74ba99-7da5-4bd8-857a-eeaff76d2d82"}
{"level":"info","ts":"2026-07-13T16:08:32.280Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ab74ba99-7da5-4bd8-857a-eeaff76d2d82","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:32.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ab74ba99-7da5-4bd8-857a-eeaff76d2d82","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:32.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ab74ba99-7da5-4bd8-857a-eeaff76d2d82","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:32.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ab74ba99-7da5-4bd8-857a-eeaff76d2d82","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:32.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ab74ba99-7da5-4bd8-857a-eeaff76d2d82","worker Pod":"long-running-rfrb2-small-group-worker-7gn25","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-rfrb2-small-group-worker-7gn25. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:32.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ab74ba99-7da5-4bd8-857a-eeaff76d2d82","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:32.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ab74ba99-7da5-4bd8-857a-eeaff76d2d82","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:08:32.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ab74ba99-7da5-4bd8-857a-eeaff76d2d82","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:32.281Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ab74ba99-7da5-4bd8-857a-eeaff76d2d82","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:32.281Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ab74ba99-7da5-4bd8-857a-eeaff76d2d82","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:32.281Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ab74ba99-7da5-4bd8-857a-eeaff76d2d82","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:08:32.281Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ab74ba99-7da5-4bd8-857a-eeaff76d2d82","seconds":300}
{"level":"info","ts":"2026-07-13T16:08:32.581Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8e5d4cf9-acc6-41ed-b006-6edd0ac17fa0"}
{"level":"info","ts":"2026-07-13T16:08:32.581Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8e5d4cf9-acc6-41ed-b006-6edd0ac17fa0"}
{"level":"info","ts":"2026-07-13T16:08:32.581Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8e5d4cf9-acc6-41ed-b006-6edd0ac17fa0","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:32.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8e5d4cf9-acc6-41ed-b006-6edd0ac17fa0","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:32.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8e5d4cf9-acc6-41ed-b006-6edd0ac17fa0","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:32.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8e5d4cf9-acc6-41ed-b006-6edd0ac17fa0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:32.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8e5d4cf9-acc6-41ed-b006-6edd0ac17fa0","worker Pod":"long-running-rfrb2-small-group-worker-7gn25","shouldDelete":true,"reason":"The worker Pod long-running-rfrb2-small-group-worker-7gn25 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-07-13T16:08:32.585Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8e5d4cf9-acc6-41ed-b006-6edd0ac17fa0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"delete 1 unhealthy worker Pods","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:32.585Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8e5d4cf9-acc6-41ed-b006-6edd0ac17fa0","error":"delete 1 unhealthy worker Pods"}
{"level":"info","ts":"2026-07-13T16:08:32.585Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8e5d4cf9-acc6-41ed-b006-6edd0ac17fa0","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:32.585Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8e5d4cf9-acc6-41ed-b006-6edd0ac17fa0"}
{"level":"error","ts":"2026-07-13T16:08:32.585Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"8e5d4cf9-acc6-41ed-b006-6edd0ac17fa0","error":"delete 1 unhealthy worker Pods","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:32.589Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6d8892c6-a552-4c38-b60e-ef1056986dc2"}
{"level":"info","ts":"2026-07-13T16:08:32.589Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6d8892c6-a552-4c38-b60e-ef1056986dc2"}
{"level":"info","ts":"2026-07-13T16:08:32.589Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6d8892c6-a552-4c38-b60e-ef1056986dc2","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:32.589Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6d8892c6-a552-4c38-b60e-ef1056986dc2","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:32.589Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6d8892c6-a552-4c38-b60e-ef1056986dc2","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:32.589Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6d8892c6-a552-4c38-b60e-ef1056986dc2","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:08:32.589Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6d8892c6-a552-4c38-b60e-ef1056986dc2","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:32.589Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6d8892c6-a552-4c38-b60e-ef1056986dc2","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:32.590Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6d8892c6-a552-4c38-b60e-ef1056986dc2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:08:32.623Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6d8892c6-a552-4c38-b60e-ef1056986dc2","seconds":300}
{"level":"info","ts":"2026-07-13T16:08:32.623Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb"}
{"level":"info","ts":"2026-07-13T16:08:32.623Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb"}
{"level":"info","ts":"2026-07-13T16:08:32.623Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:32.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:32.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:32.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:32.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:32.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:08:32.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:32.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:08:32.623Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb"}
{"level":"info","ts":"2026-07-13T16:08:32.623Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb","nodeType":"worker","rayStartParams":{"address":"long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:08:32.623Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb","rayStartCmd":"ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:32.623Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:32.623Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb","enabled":true}
{"level":"error","ts":"2026-07-13T16:08:32.627Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:32.627Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:08:32.627Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:32.627Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:08:32Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}]}
{"level":"info","ts":"2026-07-13T16:08:32.627Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb","name":"long-running-rfrb2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:08:32Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:08:32.722Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb"}
{"level":"error","ts":"2026-07-13T16:08:32.722Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"326b210d-2d56-40cc-ab6f-593fd6ef3ecb","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:32.722Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db"}
{"level":"info","ts":"2026-07-13T16:08:32.722Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db"}
{"level":"info","ts":"2026-07-13T16:08:32.722Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:32.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:32.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:32.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:32.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:32.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:08:32.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:32.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:08:32.722Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db"}
{"level":"info","ts":"2026-07-13T16:08:32.722Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db","nodeType":"worker","rayStartParams":{"address":"long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:08:32.722Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db","rayStartCmd":"ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:32.722Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:32.722Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db","enabled":true}
{"level":"info","ts":"2026-07-13T16:08:32.722Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"4f732957-5d98-4b22-8e0d-deb1638b169e","rayCluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:32.723Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"dad88ee8-24a0-4825-be12-038e690aff0b","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:32.723Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"466a6ef5-811c-4c85-81d5-741a56dfbfc6","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:32.723Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"466a6ef5-811c-4c85-81d5-741a56dfbfc6","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-07-13 16:08:17 +0000 UTC","Now":"2026-07-13T16:08:32.723Z","ShutdownTime":"2026-07-13T16:18:17.000Z"}
{"level":"info","ts":"2026-07-13T16:08:32.723Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"4f732957-5d98-4b22-8e0d-deb1638b169e","pod":"long-running-rfrb2-head-q8chr","ip":"10.133.0.38"}
{"level":"info","ts":"2026-07-13T16:08:32.723Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"466a6ef5-811c-4c85-81d5-741a56dfbfc6","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T16:08:32.723Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"466a6ef5-811c-4c85-81d5-741a56dfbfc6","seconds":586}
{"level":"error","ts":"2026-07-13T16:08:32.725Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:32.725Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:08:32.726Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:32.726Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:08:32Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}]}
{"level":"info","ts":"2026-07-13T16:08:32.726Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db","name":"long-running-rfrb2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:08:32Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:08:32.727Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"dad88ee8-24a0-4825-be12-038e690aff0b","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:32.821Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db","name":"long-running-rfrb2","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-rfrb2\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-x5qvg","name":"long-running-rfrb2"}}
{"level":"info","ts":"2026-07-13T16:08:32.821Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db"}
{"level":"error","ts":"2026-07-13T16:08:32.821Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d61fb6a0-07b5-45d2-85bc-1224146668db","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:32.821Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"4f732957-5d98-4b22-8e0d-deb1638b169e","certificate":"ray-head-cert-long-running-rfrb2","dnsNames":["long-running-rfrb2-head-svc","localhost","long-running-rfrb2-head-svc.test-ns-x5qvg.svc","long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local"],"ipAddresses":["10.133.0.38","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:08:32.821Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7a115ff8-608f-44db-a9f5-8cfb7b15aac4"}
{"level":"info","ts":"2026-07-13T16:08:32.821Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7a115ff8-608f-44db-a9f5-8cfb7b15aac4"}
{"level":"info","ts":"2026-07-13T16:08:32.822Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7a115ff8-608f-44db-a9f5-8cfb7b15aac4","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:32.822Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7a115ff8-608f-44db-a9f5-8cfb7b15aac4","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:32.822Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7a115ff8-608f-44db-a9f5-8cfb7b15aac4","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:32.822Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7a115ff8-608f-44db-a9f5-8cfb7b15aac4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:32.822Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7a115ff8-608f-44db-a9f5-8cfb7b15aac4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:32.822Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7a115ff8-608f-44db-a9f5-8cfb7b15aac4","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:08:32.822Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7a115ff8-608f-44db-a9f5-8cfb7b15aac4","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:32.822Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7a115ff8-608f-44db-a9f5-8cfb7b15aac4","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:08:32.822Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7a115ff8-608f-44db-a9f5-8cfb7b15aac4"}
{"level":"info","ts":"2026-07-13T16:08:32.822Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7a115ff8-608f-44db-a9f5-8cfb7b15aac4","nodeType":"worker","rayStartParams":{"address":"long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:08:32.822Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7a115ff8-608f-44db-a9f5-8cfb7b15aac4","rayStartCmd":"ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:32.822Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7a115ff8-608f-44db-a9f5-8cfb7b15aac4","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:32.822Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7a115ff8-608f-44db-a9f5-8cfb7b15aac4","enabled":true}
{"level":"error","ts":"2026-07-13T16:08:32.825Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7a115ff8-608f-44db-a9f5-8cfb7b15aac4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:32.825Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7a115ff8-608f-44db-a9f5-8cfb7b15aac4","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:08:32.825Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7a115ff8-608f-44db-a9f5-8cfb7b15aac4","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:32.825Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7a115ff8-608f-44db-a9f5-8cfb7b15aac4"}
{"level":"error","ts":"2026-07-13T16:08:32.825Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"7a115ff8-608f-44db-a9f5-8cfb7b15aac4","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:32.832Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"30bb4ec0-bd2c-45af-af5f-d9ba5b571469"}
{"level":"info","ts":"2026-07-13T16:08:32.832Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"30bb4ec0-bd2c-45af-af5f-d9ba5b571469"}
{"level":"info","ts":"2026-07-13T16:08:32.832Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"30bb4ec0-bd2c-45af-af5f-d9ba5b571469","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:32.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"30bb4ec0-bd2c-45af-af5f-d9ba5b571469","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:32.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"30bb4ec0-bd2c-45af-af5f-d9ba5b571469","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:32.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"30bb4ec0-bd2c-45af-af5f-d9ba5b571469","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:32.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"30bb4ec0-bd2c-45af-af5f-d9ba5b571469","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:32.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"30bb4ec0-bd2c-45af-af5f-d9ba5b571469","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:08:32.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"30bb4ec0-bd2c-45af-af5f-d9ba5b571469","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:32.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"30bb4ec0-bd2c-45af-af5f-d9ba5b571469","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:08:32.832Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"30bb4ec0-bd2c-45af-af5f-d9ba5b571469"}
{"level":"info","ts":"2026-07-13T16:08:32.832Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"30bb4ec0-bd2c-45af-af5f-d9ba5b571469","nodeType":"worker","rayStartParams":{"address":"long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:08:32.832Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"30bb4ec0-bd2c-45af-af5f-d9ba5b571469","rayStartCmd":"ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:32.832Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"30bb4ec0-bd2c-45af-af5f-d9ba5b571469","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:32.832Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"30bb4ec0-bd2c-45af-af5f-d9ba5b571469","enabled":true}
{"level":"info","ts":"2026-07-13T16:08:32.834Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"4f732957-5d98-4b22-8e0d-deb1638b169e","certificate":"ray-worker-cert-long-running-rfrb2","dnsNames":["long-running-rfrb2-worker-svc","localhost","long-running-rfrb2-small-group","long-running-rfrb2-worker-svc.test-ns-x5qvg.svc","long-running-rfrb2-worker-svc.test-ns-x5qvg.svc.cluster.local","*.long-running-rfrb2-worker-svc.test-ns-x5qvg.svc","*.long-running-rfrb2-worker-svc.test-ns-x5qvg.svc.cluster.local","*-worker-*.test-ns-x5qvg.svc","*-worker-*.test-ns-x5qvg.svc.cluster.local"],"ipAddresses":["10.133.0.38","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:08:32.834Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"4f732957-5d98-4b22-8e0d-deb1638b169e"}
{"level":"error","ts":"2026-07-13T16:08:32.835Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"30bb4ec0-bd2c-45af-af5f-d9ba5b571469","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:32.835Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"30bb4ec0-bd2c-45af-af5f-d9ba5b571469","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:08:32.835Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"30bb4ec0-bd2c-45af-af5f-d9ba5b571469","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:32.835Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"30bb4ec0-bd2c-45af-af5f-d9ba5b571469"}
{"level":"error","ts":"2026-07-13T16:08:32.835Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"30bb4ec0-bd2c-45af-af5f-d9ba5b571469","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:32.921Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a4684505-2213-48f4-ab91-d9b2c5069ea0"}
{"level":"info","ts":"2026-07-13T16:08:32.921Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a4684505-2213-48f4-ab91-d9b2c5069ea0"}
{"level":"info","ts":"2026-07-13T16:08:32.921Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a4684505-2213-48f4-ab91-d9b2c5069ea0","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:32.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a4684505-2213-48f4-ab91-d9b2c5069ea0","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:32.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a4684505-2213-48f4-ab91-d9b2c5069ea0","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:32.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a4684505-2213-48f4-ab91-d9b2c5069ea0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:32.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a4684505-2213-48f4-ab91-d9b2c5069ea0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:32.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a4684505-2213-48f4-ab91-d9b2c5069ea0","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:08:32.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a4684505-2213-48f4-ab91-d9b2c5069ea0","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:32.921Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a4684505-2213-48f4-ab91-d9b2c5069ea0","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:08:32.921Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a4684505-2213-48f4-ab91-d9b2c5069ea0"}
{"level":"info","ts":"2026-07-13T16:08:32.921Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a4684505-2213-48f4-ab91-d9b2c5069ea0","nodeType":"worker","rayStartParams":{"address":"long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:08:32.921Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a4684505-2213-48f4-ab91-d9b2c5069ea0","rayStartCmd":"ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:32.921Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a4684505-2213-48f4-ab91-d9b2c5069ea0","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:32.921Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a4684505-2213-48f4-ab91-d9b2c5069ea0","enabled":true}
{"level":"error","ts":"2026-07-13T16:08:32.925Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a4684505-2213-48f4-ab91-d9b2c5069ea0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:32.925Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a4684505-2213-48f4-ab91-d9b2c5069ea0","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:08:32.926Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a4684505-2213-48f4-ab91-d9b2c5069ea0","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:32.926Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a4684505-2213-48f4-ab91-d9b2c5069ea0"}
{"level":"error","ts":"2026-07-13T16:08:32.926Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"a4684505-2213-48f4-ab91-d9b2c5069ea0","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:33.006Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"77850ce0-5a37-4ebc-b624-f12a575d8c3e"}
{"level":"info","ts":"2026-07-13T16:08:33.006Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"77850ce0-5a37-4ebc-b624-f12a575d8c3e"}
{"level":"info","ts":"2026-07-13T16:08:33.006Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"77850ce0-5a37-4ebc-b624-f12a575d8c3e","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:33.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"77850ce0-5a37-4ebc-b624-f12a575d8c3e","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:33.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"77850ce0-5a37-4ebc-b624-f12a575d8c3e","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:33.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"77850ce0-5a37-4ebc-b624-f12a575d8c3e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:33.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"77850ce0-5a37-4ebc-b624-f12a575d8c3e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:33.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"77850ce0-5a37-4ebc-b624-f12a575d8c3e","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:08:33.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"77850ce0-5a37-4ebc-b624-f12a575d8c3e","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:33.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"77850ce0-5a37-4ebc-b624-f12a575d8c3e","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:08:33.007Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"77850ce0-5a37-4ebc-b624-f12a575d8c3e"}
{"level":"info","ts":"2026-07-13T16:08:33.007Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"77850ce0-5a37-4ebc-b624-f12a575d8c3e","nodeType":"worker","rayStartParams":{"address":"long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:08:33.007Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"77850ce0-5a37-4ebc-b624-f12a575d8c3e","rayStartCmd":"ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:33.007Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"77850ce0-5a37-4ebc-b624-f12a575d8c3e","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:33.007Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"77850ce0-5a37-4ebc-b624-f12a575d8c3e","enabled":true}
{"level":"error","ts":"2026-07-13T16:08:33.010Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"77850ce0-5a37-4ebc-b624-f12a575d8c3e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:33.010Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"77850ce0-5a37-4ebc-b624-f12a575d8c3e","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:08:33.010Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"77850ce0-5a37-4ebc-b624-f12a575d8c3e","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:33.010Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"77850ce0-5a37-4ebc-b624-f12a575d8c3e"}
{"level":"error","ts":"2026-07-13T16:08:33.010Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"77850ce0-5a37-4ebc-b624-f12a575d8c3e","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:33.171Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1038567a-10a9-4a47-9e69-278b5c493744"}
{"level":"info","ts":"2026-07-13T16:08:33.172Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1038567a-10a9-4a47-9e69-278b5c493744"}
{"level":"info","ts":"2026-07-13T16:08:33.172Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1038567a-10a9-4a47-9e69-278b5c493744","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:33.172Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1038567a-10a9-4a47-9e69-278b5c493744","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:33.172Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1038567a-10a9-4a47-9e69-278b5c493744","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:33.172Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1038567a-10a9-4a47-9e69-278b5c493744","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:33.172Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1038567a-10a9-4a47-9e69-278b5c493744","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:33.172Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1038567a-10a9-4a47-9e69-278b5c493744","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:08:33.172Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1038567a-10a9-4a47-9e69-278b5c493744","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:33.172Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1038567a-10a9-4a47-9e69-278b5c493744","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:08:33.172Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1038567a-10a9-4a47-9e69-278b5c493744"}
{"level":"info","ts":"2026-07-13T16:08:33.172Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1038567a-10a9-4a47-9e69-278b5c493744","nodeType":"worker","rayStartParams":{"address":"long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:08:33.172Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1038567a-10a9-4a47-9e69-278b5c493744","rayStartCmd":"ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:33.172Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1038567a-10a9-4a47-9e69-278b5c493744","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:33.172Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1038567a-10a9-4a47-9e69-278b5c493744","enabled":true}
{"level":"error","ts":"2026-07-13T16:08:33.175Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1038567a-10a9-4a47-9e69-278b5c493744","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:33.175Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1038567a-10a9-4a47-9e69-278b5c493744","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:08:33.175Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1038567a-10a9-4a47-9e69-278b5c493744","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:33.175Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1038567a-10a9-4a47-9e69-278b5c493744"}
{"level":"error","ts":"2026-07-13T16:08:33.175Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"1038567a-10a9-4a47-9e69-278b5c493744","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:33.496Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b554ee69-c00a-46d2-98d7-c1428f15a265"}
{"level":"info","ts":"2026-07-13T16:08:33.496Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b554ee69-c00a-46d2-98d7-c1428f15a265"}
{"level":"info","ts":"2026-07-13T16:08:33.496Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b554ee69-c00a-46d2-98d7-c1428f15a265","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:33.496Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b554ee69-c00a-46d2-98d7-c1428f15a265","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:33.496Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b554ee69-c00a-46d2-98d7-c1428f15a265","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:33.496Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b554ee69-c00a-46d2-98d7-c1428f15a265","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:33.496Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b554ee69-c00a-46d2-98d7-c1428f15a265","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:33.496Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b554ee69-c00a-46d2-98d7-c1428f15a265","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:08:33.496Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b554ee69-c00a-46d2-98d7-c1428f15a265","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:33.496Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b554ee69-c00a-46d2-98d7-c1428f15a265","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:08:33.496Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b554ee69-c00a-46d2-98d7-c1428f15a265"}
{"level":"info","ts":"2026-07-13T16:08:33.496Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b554ee69-c00a-46d2-98d7-c1428f15a265","nodeType":"worker","rayStartParams":{"address":"long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:08:33.496Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b554ee69-c00a-46d2-98d7-c1428f15a265","rayStartCmd":"ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:33.496Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b554ee69-c00a-46d2-98d7-c1428f15a265","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:33.496Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b554ee69-c00a-46d2-98d7-c1428f15a265","enabled":true}
{"level":"error","ts":"2026-07-13T16:08:33.500Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b554ee69-c00a-46d2-98d7-c1428f15a265","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:33.500Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b554ee69-c00a-46d2-98d7-c1428f15a265","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:08:33.500Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b554ee69-c00a-46d2-98d7-c1428f15a265","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:33.500Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b554ee69-c00a-46d2-98d7-c1428f15a265"}
{"level":"error","ts":"2026-07-13T16:08:33.500Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"b554ee69-c00a-46d2-98d7-c1428f15a265","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:34.140Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"70ce8e24-ae78-4528-8991-746952986c25"}
{"level":"info","ts":"2026-07-13T16:08:34.140Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"70ce8e24-ae78-4528-8991-746952986c25"}
{"level":"info","ts":"2026-07-13T16:08:34.140Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"70ce8e24-ae78-4528-8991-746952986c25","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:34.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"70ce8e24-ae78-4528-8991-746952986c25","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:34.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"70ce8e24-ae78-4528-8991-746952986c25","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:34.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"70ce8e24-ae78-4528-8991-746952986c25","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:34.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"70ce8e24-ae78-4528-8991-746952986c25","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:34.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"70ce8e24-ae78-4528-8991-746952986c25","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:08:34.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"70ce8e24-ae78-4528-8991-746952986c25","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:34.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"70ce8e24-ae78-4528-8991-746952986c25","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:08:34.140Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"70ce8e24-ae78-4528-8991-746952986c25"}
{"level":"info","ts":"2026-07-13T16:08:34.140Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"70ce8e24-ae78-4528-8991-746952986c25","nodeType":"worker","rayStartParams":{"address":"long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:08:34.140Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"70ce8e24-ae78-4528-8991-746952986c25","rayStartCmd":"ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:34.140Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"70ce8e24-ae78-4528-8991-746952986c25","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:34.140Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"70ce8e24-ae78-4528-8991-746952986c25","enabled":true}
{"level":"error","ts":"2026-07-13T16:08:34.145Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"70ce8e24-ae78-4528-8991-746952986c25","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:34.145Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"70ce8e24-ae78-4528-8991-746952986c25","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:08:34.145Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"70ce8e24-ae78-4528-8991-746952986c25","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:34.145Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"70ce8e24-ae78-4528-8991-746952986c25"}
{"level":"error","ts":"2026-07-13T16:08:34.145Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"70ce8e24-ae78-4528-8991-746952986c25","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:34.949Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"fc2a1c9d-2f0b-48cc-bd17-c6064e9e1852","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:34.949Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"fc2a1c9d-2f0b-48cc-bd17-c6064e9e1852","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T16:08:34.949Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"fc2a1c9d-2f0b-48cc-bd17-c6064e9e1852","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T16:08:34.949Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"fc2a1c9d-2f0b-48cc-bd17-c6064e9e1852","rayCluster":"raycluster","podIPs":["10.133.0.18","10.134.0.25"]}
{"level":"info","ts":"2026-07-13T16:08:35.426Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"847b6236-6e01-433d-8d11-d39903dea976"}
{"level":"info","ts":"2026-07-13T16:08:35.427Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"847b6236-6e01-433d-8d11-d39903dea976"}
{"level":"info","ts":"2026-07-13T16:08:35.427Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"847b6236-6e01-433d-8d11-d39903dea976","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:35.427Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"847b6236-6e01-433d-8d11-d39903dea976","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:35.427Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"847b6236-6e01-433d-8d11-d39903dea976","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:35.427Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"847b6236-6e01-433d-8d11-d39903dea976","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:35.427Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"847b6236-6e01-433d-8d11-d39903dea976","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:35.427Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"847b6236-6e01-433d-8d11-d39903dea976","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:08:35.427Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"847b6236-6e01-433d-8d11-d39903dea976","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:35.427Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"847b6236-6e01-433d-8d11-d39903dea976","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:08:35.427Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"847b6236-6e01-433d-8d11-d39903dea976"}
{"level":"info","ts":"2026-07-13T16:08:35.427Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"847b6236-6e01-433d-8d11-d39903dea976","nodeType":"worker","rayStartParams":{"address":"long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:08:35.427Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"847b6236-6e01-433d-8d11-d39903dea976","rayStartCmd":"ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:35.427Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"847b6236-6e01-433d-8d11-d39903dea976","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:35.427Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"847b6236-6e01-433d-8d11-d39903dea976","enabled":true}
{"level":"error","ts":"2026-07-13T16:08:35.430Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"847b6236-6e01-433d-8d11-d39903dea976","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:35.430Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"847b6236-6e01-433d-8d11-d39903dea976","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:08:35.430Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"847b6236-6e01-433d-8d11-d39903dea976","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:35.430Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"847b6236-6e01-433d-8d11-d39903dea976"}
{"level":"error","ts":"2026-07-13T16:08:35.430Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"847b6236-6e01-433d-8d11-d39903dea976","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:37.991Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6e9db0bb-1a6e-4703-b6a6-026e2690bd32"}
{"level":"info","ts":"2026-07-13T16:08:37.991Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6e9db0bb-1a6e-4703-b6a6-026e2690bd32"}
{"level":"info","ts":"2026-07-13T16:08:37.991Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6e9db0bb-1a6e-4703-b6a6-026e2690bd32","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:37.991Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6e9db0bb-1a6e-4703-b6a6-026e2690bd32","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:37.991Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6e9db0bb-1a6e-4703-b6a6-026e2690bd32","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:37.991Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6e9db0bb-1a6e-4703-b6a6-026e2690bd32","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:37.991Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6e9db0bb-1a6e-4703-b6a6-026e2690bd32","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:37.991Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6e9db0bb-1a6e-4703-b6a6-026e2690bd32","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:08:37.991Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6e9db0bb-1a6e-4703-b6a6-026e2690bd32","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:37.991Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6e9db0bb-1a6e-4703-b6a6-026e2690bd32","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:08:37.991Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6e9db0bb-1a6e-4703-b6a6-026e2690bd32"}
{"level":"info","ts":"2026-07-13T16:08:37.991Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6e9db0bb-1a6e-4703-b6a6-026e2690bd32","nodeType":"worker","rayStartParams":{"address":"long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:08:37.991Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6e9db0bb-1a6e-4703-b6a6-026e2690bd32","rayStartCmd":"ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:37.991Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6e9db0bb-1a6e-4703-b6a6-026e2690bd32","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:37.991Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6e9db0bb-1a6e-4703-b6a6-026e2690bd32","enabled":true}
{"level":"error","ts":"2026-07-13T16:08:37.995Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6e9db0bb-1a6e-4703-b6a6-026e2690bd32","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:37.995Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6e9db0bb-1a6e-4703-b6a6-026e2690bd32","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:08:37.995Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6e9db0bb-1a6e-4703-b6a6-026e2690bd32","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:37.995Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6e9db0bb-1a6e-4703-b6a6-026e2690bd32"}
{"level":"error","ts":"2026-07-13T16:08:37.995Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"6e9db0bb-1a6e-4703-b6a6-026e2690bd32","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:39.103Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"705328c1-e80f-4a8d-ad6f-b85fe3febe45","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:39.103Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"705328c1-e80f-4a8d-ad6f-b85fe3febe45","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:39.103Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"705328c1-e80f-4a8d-ad6f-b85fe3febe45","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:08:39.110Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"705328c1-e80f-4a8d-ad6f-b85fe3febe45","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-hd5d2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T16:08:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:08:29Z","endTime":"2026-07-13T16:08:33Z"},"jobId":"counter-hd5d2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T16:08:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:08:39.111Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"705328c1-e80f-4a8d-ad6f-b85fe3febe45","old JobStatus":"","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T16:08:39.124Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"d9610162-4ffa-4952-8dfb-9d0dd5f72853","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:39.124Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"d9610162-4ffa-4952-8dfb-9d0dd5f72853","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:39.124Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"d9610162-4ffa-4952-8dfb-9d0dd5f72853","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:08:39.126Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"d9610162-4ffa-4952-8dfb-9d0dd5f72853","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:08:29Z","endTime":"2026-07-13T16:08:33Z"},"jobId":"counter-hd5d2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T16:08:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:08:29Z","endTime":"2026-07-13T16:08:33Z"},"jobId":"counter-hd5d2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T16:08:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:08:39.208Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"83a2cb94-741f-42f8-b4d4-a3aec7eb1701","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:39.208Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"83a2cb94-741f-42f8-b4d4-a3aec7eb1701","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T16:08:39.216Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"83a2cb94-741f-42f8-b4d4-a3aec7eb1701"}
{"level":"info","ts":"2026-07-13T16:08:39.216Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"83a2cb94-741f-42f8-b4d4-a3aec7eb1701","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-07-13T16:08:39.216Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"83a2cb94-741f-42f8-b4d4-a3aec7eb1701","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-5jsfr","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T16:08:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T16:08:39.216Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"83a2cb94-741f-42f8-b4d4-a3aec7eb1701","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-07-13T16:08:39.227Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"921b545f-1a01-41c8-9532-38fafb11d224","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:39.227Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"921b545f-1a01-41c8-9532-38fafb11d224","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:39.227Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"921b545f-1a01-41c8-9532-38fafb11d224","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:08:39.227Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"921b545f-1a01-41c8-9532-38fafb11d224","head service name":"raycluster-head-svc","namespace":"test-ns-qbjjb"}
{"level":"info","ts":"2026-07-13T16:08:39.227Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"921b545f-1a01-41c8-9532-38fafb11d224","head service URL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265"}
{"level":"info","ts":"2026-07-13T16:08:39.227Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"921b545f-1a01-41c8-9532-38fafb11d224"}
{"level":"info","ts":"2026-07-13T16:08:39.227Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"921b545f-1a01-41c8-9532-38fafb11d224","command":["if","!","ray","job","status","--address","http://raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","fail-5jsfr",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","--no-wait","--submission-id","fail-5jsfr","--","python /home/ray/jobs/fail.py",";","fi",";","ray","job","logs","--address","http://raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","--follow","fail-5jsfr"]}
{"level":"info","ts":"2026-07-13T16:08:39.243Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"921b545f-1a01-41c8-9532-38fafb11d224","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-07-13T16:08:39.243Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"921b545f-1a01-41c8-9532-38fafb11d224","SubmissionMode":"K8sJobMode","RayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:39.243Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"921b545f-1a01-41c8-9532-38fafb11d224","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-5jsfr","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-07-13T16:08:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-5jsfr","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T16:08:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-07-13T16:08:39.243Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"921b545f-1a01-41c8-9532-38fafb11d224","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T16:08:39.252Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"c025b12d-cb4d-4098-935e-9d5c745a50da","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:39.252Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"c025b12d-cb4d-4098-935e-9d5c745a50da","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:39.252Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"c025b12d-cb4d-4098-935e-9d5c745a50da","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:39.255Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"c025b12d-cb4d-4098-935e-9d5c745a50da","JobId":"fail-5jsfr","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:39.255Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"c025b12d-cb4d-4098-935e-9d5c745a50da"}
{"level":"error","ts":"2026-07-13T16:08:39.255Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"c025b12d-cb4d-4098-935e-9d5c745a50da","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:39.255Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"096b7121-0dd4-43cc-8ccd-b114c1309536","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:39.255Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"096b7121-0dd4-43cc-8ccd-b114c1309536","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:39.255Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"096b7121-0dd4-43cc-8ccd-b114c1309536","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:39.260Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"096b7121-0dd4-43cc-8ccd-b114c1309536","JobId":"fail-5jsfr","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:39.260Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"096b7121-0dd4-43cc-8ccd-b114c1309536"}
{"level":"error","ts":"2026-07-13T16:08:39.260Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"096b7121-0dd4-43cc-8ccd-b114c1309536","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:39.260Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"bda1e33f-a62b-4dd1-9beb-59e6dc4af455","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:39.260Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"bda1e33f-a62b-4dd1-9beb-59e6dc4af455","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:39.260Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"bda1e33f-a62b-4dd1-9beb-59e6dc4af455","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:39.265Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"bda1e33f-a62b-4dd1-9beb-59e6dc4af455","JobId":"fail-5jsfr","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:39.265Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"bda1e33f-a62b-4dd1-9beb-59e6dc4af455"}
{"level":"error","ts":"2026-07-13T16:08:39.265Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"bda1e33f-a62b-4dd1-9beb-59e6dc4af455","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:39.265Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"b4229cac-3b87-4252-88b8-d9c765b7cc7b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:39.265Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"b4229cac-3b87-4252-88b8-d9c765b7cc7b","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:39.265Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"b4229cac-3b87-4252-88b8-d9c765b7cc7b","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:39.270Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"b4229cac-3b87-4252-88b8-d9c765b7cc7b","JobId":"fail-5jsfr","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:39.270Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"b4229cac-3b87-4252-88b8-d9c765b7cc7b"}
{"level":"error","ts":"2026-07-13T16:08:39.270Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"b4229cac-3b87-4252-88b8-d9c765b7cc7b","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:39.271Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"d053a46c-affe-458f-bfa8-82990b900bbf","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:39.271Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"d053a46c-affe-458f-bfa8-82990b900bbf","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:39.271Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"d053a46c-affe-458f-bfa8-82990b900bbf","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:39.282Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"d053a46c-affe-458f-bfa8-82990b900bbf","JobId":"fail-5jsfr","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:39.282Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"d053a46c-affe-458f-bfa8-82990b900bbf"}
{"level":"error","ts":"2026-07-13T16:08:39.282Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"d053a46c-affe-458f-bfa8-82990b900bbf","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:39.362Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"43b4d0ab-c440-4f4b-8cb1-917372b3d811","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:39.362Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"43b4d0ab-c440-4f4b-8cb1-917372b3d811","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:39.362Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"43b4d0ab-c440-4f4b-8cb1-917372b3d811","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:39.367Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"43b4d0ab-c440-4f4b-8cb1-917372b3d811","JobId":"fail-5jsfr","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:39.367Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"43b4d0ab-c440-4f4b-8cb1-917372b3d811"}
{"level":"error","ts":"2026-07-13T16:08:39.367Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"43b4d0ab-c440-4f4b-8cb1-917372b3d811","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:39.528Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"7a72c422-0df6-4ba4-a499-a3c5372f7850","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:39.528Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"7a72c422-0df6-4ba4-a499-a3c5372f7850","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:39.528Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"7a72c422-0df6-4ba4-a499-a3c5372f7850","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:39.533Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"7a72c422-0df6-4ba4-a499-a3c5372f7850","JobId":"fail-5jsfr","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:39.533Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"7a72c422-0df6-4ba4-a499-a3c5372f7850"}
{"level":"error","ts":"2026-07-13T16:08:39.533Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"7a72c422-0df6-4ba4-a499-a3c5372f7850","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:39.854Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"f9b0b571-fdef-46da-bfa3-6b21d68b4617","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:39.855Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"f9b0b571-fdef-46da-bfa3-6b21d68b4617","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:39.855Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"f9b0b571-fdef-46da-bfa3-6b21d68b4617","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:39.860Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"f9b0b571-fdef-46da-bfa3-6b21d68b4617","JobId":"fail-5jsfr","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:39.860Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"f9b0b571-fdef-46da-bfa3-6b21d68b4617"}
{"level":"error","ts":"2026-07-13T16:08:39.860Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"f9b0b571-fdef-46da-bfa3-6b21d68b4617","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:40.501Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"55fc88a2-f2e4-4c57-a0e2-b2d858d43d66","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:40.501Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"55fc88a2-f2e4-4c57-a0e2-b2d858d43d66","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:40.501Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"55fc88a2-f2e4-4c57-a0e2-b2d858d43d66","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:40.506Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"55fc88a2-f2e4-4c57-a0e2-b2d858d43d66","JobId":"fail-5jsfr","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:40.506Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"55fc88a2-f2e4-4c57-a0e2-b2d858d43d66"}
{"level":"error","ts":"2026-07-13T16:08:40.506Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"55fc88a2-f2e4-4c57-a0e2-b2d858d43d66","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:41.247Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"b5fda53d-7c6a-4281-8993-b4887c7cad54","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:41.247Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"b5fda53d-7c6a-4281-8993-b4887c7cad54","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:41.247Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"b5fda53d-7c6a-4281-8993-b4887c7cad54","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:08:41.254Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"b5fda53d-7c6a-4281-8993-b4887c7cad54","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:08:29Z","endTime":"2026-07-13T16:08:33Z"},"jobId":"counter-hd5d2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T16:08:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:08:29Z","endTime":"2026-07-13T16:08:33Z"},"jobId":"counter-hd5d2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T16:08:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:08:41.609Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"00536968-a792-4120-a623-ec7774134f9d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:41.609Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"00536968-a792-4120-a623-ec7774134f9d","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:41.609Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"00536968-a792-4120-a623-ec7774134f9d","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:41.612Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"00536968-a792-4120-a623-ec7774134f9d","JobId":"fail-5jsfr","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:41.612Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"00536968-a792-4120-a623-ec7774134f9d"}
{"level":"error","ts":"2026-07-13T16:08:41.612Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"00536968-a792-4120-a623-ec7774134f9d","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:41.786Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"be5b23d2-6df9-480c-b075-3515a51bd651","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:41.786Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"be5b23d2-6df9-480c-b075-3515a51bd651","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:41.786Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"be5b23d2-6df9-480c-b075-3515a51bd651","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:41.792Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"be5b23d2-6df9-480c-b075-3515a51bd651","JobId":"fail-5jsfr","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:41.792Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"be5b23d2-6df9-480c-b075-3515a51bd651"}
{"level":"error","ts":"2026-07-13T16:08:41.792Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"be5b23d2-6df9-480c-b075-3515a51bd651","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:42.123Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"324a87cb-78db-4e08-8ab9-dc85d01aa0ce","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:42.123Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"324a87cb-78db-4e08-8ab9-dc85d01aa0ce","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:42.123Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"324a87cb-78db-4e08-8ab9-dc85d01aa0ce","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:08:42.129Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"324a87cb-78db-4e08-8ab9-dc85d01aa0ce","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:08:29Z","endTime":"2026-07-13T16:08:33Z"},"jobId":"counter-hd5d2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T16:08:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:08:29Z","endTime":"2026-07-13T16:08:33Z"},"jobId":"counter-hd5d2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T16:08:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:08:42.254Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"23a5d37b-ee1f-4688-802f-9f16df9575cc","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:42.254Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"23a5d37b-ee1f-4688-802f-9f16df9575cc","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:42.254Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"23a5d37b-ee1f-4688-802f-9f16df9575cc","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:08:42.257Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"23a5d37b-ee1f-4688-802f-9f16df9575cc","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:08:29Z","endTime":"2026-07-13T16:08:33Z"},"jobId":"counter-hd5d2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T16:08:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:08:29Z","endTime":"2026-07-13T16:08:33Z"},"jobId":"counter-hd5d2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T16:08:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:08:42.270Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"93ac829d-d883-4aca-a9e4-d33eddf7cc61","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:42.270Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"93ac829d-d883-4aca-a9e4-d33eddf7cc61","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:42.270Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"93ac829d-d883-4aca-a9e4-d33eddf7cc61","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:08:42.273Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"93ac829d-d883-4aca-a9e4-d33eddf7cc61","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:08:29Z","endTime":"2026-07-13T16:08:33Z"},"jobId":"counter-hd5d2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-07-13T16:08:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:08:29Z","endTime":"2026-07-13T16:08:33Z"},"jobId":"counter-hd5d2","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-07-13T16:08:21Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:08:42.273Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"93ac829d-d883-4aca-a9e4-d33eddf7cc61","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-07-13T16:08:42.284Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"bee073b8-52a9-43c9-a60f-1fac802872a6","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:42.284Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"bee073b8-52a9-43c9-a60f-1fac802872a6","ShutdownAfterJobFinishes":false,"ClusterSelector":{"ray.io/cluster":"raycluster"},"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 16:08:42 +0000 UTC","Now":"2026-07-13T16:08:42.284Z","ShutdownTime":"2026-07-13T16:08:42.000Z"}
{"level":"info","ts":"2026-07-13T16:08:43.116Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ce0f425d-78d5-4983-83ba-4b340d89543d"}
{"level":"info","ts":"2026-07-13T16:08:43.116Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ce0f425d-78d5-4983-83ba-4b340d89543d"}
{"level":"info","ts":"2026-07-13T16:08:43.116Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ce0f425d-78d5-4983-83ba-4b340d89543d","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:43.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ce0f425d-78d5-4983-83ba-4b340d89543d","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:43.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ce0f425d-78d5-4983-83ba-4b340d89543d","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:43.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ce0f425d-78d5-4983-83ba-4b340d89543d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:43.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ce0f425d-78d5-4983-83ba-4b340d89543d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:43.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ce0f425d-78d5-4983-83ba-4b340d89543d","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:08:43.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ce0f425d-78d5-4983-83ba-4b340d89543d","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:43.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ce0f425d-78d5-4983-83ba-4b340d89543d","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:08:43.116Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ce0f425d-78d5-4983-83ba-4b340d89543d"}
{"level":"info","ts":"2026-07-13T16:08:43.116Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ce0f425d-78d5-4983-83ba-4b340d89543d","nodeType":"worker","rayStartParams":{"address":"long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:08:43.116Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ce0f425d-78d5-4983-83ba-4b340d89543d","rayStartCmd":"ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:43.116Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ce0f425d-78d5-4983-83ba-4b340d89543d","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:43.116Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ce0f425d-78d5-4983-83ba-4b340d89543d","enabled":true}
{"level":"error","ts":"2026-07-13T16:08:43.119Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ce0f425d-78d5-4983-83ba-4b340d89543d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:43.119Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ce0f425d-78d5-4983-83ba-4b340d89543d","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:08:43.119Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ce0f425d-78d5-4983-83ba-4b340d89543d","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:43.119Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ce0f425d-78d5-4983-83ba-4b340d89543d"}
{"level":"error","ts":"2026-07-13T16:08:43.119Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ce0f425d-78d5-4983-83ba-4b340d89543d","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:43.328Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"6a843849-71eb-40ef-a091-5444ebf30429","rayCluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:43.328Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"6a843849-71eb-40ef-a091-5444ebf30429","pod":"long-running-rfrb2-head-q8chr","ip":"10.133.0.38"}
{"level":"info","ts":"2026-07-13T16:08:43.328Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"6a843849-71eb-40ef-a091-5444ebf30429"}
{"level":"info","ts":"2026-07-13T16:08:45.130Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"c7f886c3-239d-49a6-b320-45068ebe9a3a","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:45.130Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"c7f886c3-239d-49a6-b320-45068ebe9a3a","ShutdownAfterJobFinishes":false,"ClusterSelector":{"ray.io/cluster":"raycluster"},"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 16:08:42 +0000 UTC","Now":"2026-07-13T16:08:45.130Z","ShutdownTime":"2026-07-13T16:08:42.000Z"}
{"level":"info","ts":"2026-07-13T16:08:46.913Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"ce0eef48-9608-40be-a5db-2d1474067304","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:46.913Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"ce0eef48-9608-40be-a5db-2d1474067304","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:46.913Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"ce0eef48-9608-40be-a5db-2d1474067304","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"error","ts":"2026-07-13T16:08:46.916Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"ce0eef48-9608-40be-a5db-2d1474067304","JobId":"fail-5jsfr","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:46.916Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"ce0eef48-9608-40be-a5db-2d1474067304"}
{"level":"error","ts":"2026-07-13T16:08:46.916Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"ce0eef48-9608-40be-a5db-2d1474067304","error":"Job fail-5jsfr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:08:49.065Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"d0500339-6808-4cd1-98f2-90f3c7d27ca6","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:49.065Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"d0500339-6808-4cd1-98f2-90f3c7d27ca6"}
{"level":"error","ts":"2026-07-13T16:08:49.065Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"d0500339-6808-4cd1-98f2-90f3c7d27ca6","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:53.361Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ef659015-9c54-475a-8b5e-395688ec31d6"}
{"level":"info","ts":"2026-07-13T16:08:53.361Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ef659015-9c54-475a-8b5e-395688ec31d6"}
{"level":"info","ts":"2026-07-13T16:08:53.361Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ef659015-9c54-475a-8b5e-395688ec31d6","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:53.361Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ef659015-9c54-475a-8b5e-395688ec31d6","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:08:53.361Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ef659015-9c54-475a-8b5e-395688ec31d6","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-rfrb2-head-q8chr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:08:53.361Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ef659015-9c54-475a-8b5e-395688ec31d6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:53.361Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ef659015-9c54-475a-8b5e-395688ec31d6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:53.361Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ef659015-9c54-475a-8b5e-395688ec31d6","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:08:53.361Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ef659015-9c54-475a-8b5e-395688ec31d6","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:53.361Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ef659015-9c54-475a-8b5e-395688ec31d6","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:08:53.361Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ef659015-9c54-475a-8b5e-395688ec31d6"}
{"level":"info","ts":"2026-07-13T16:08:53.361Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ef659015-9c54-475a-8b5e-395688ec31d6","nodeType":"worker","rayStartParams":{"address":"long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:08:53.361Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ef659015-9c54-475a-8b5e-395688ec31d6","rayStartCmd":"ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:53.361Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ef659015-9c54-475a-8b5e-395688ec31d6","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:53.361Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ef659015-9c54-475a-8b5e-395688ec31d6","enabled":true}
{"level":"error","ts":"2026-07-13T16:08:53.364Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ef659015-9c54-475a-8b5e-395688ec31d6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:53.364Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ef659015-9c54-475a-8b5e-395688ec31d6","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:08:53.364Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ef659015-9c54-475a-8b5e-395688ec31d6","podName":"long-running-rfrb2-head-q8chr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:08:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:08:53.364Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ef659015-9c54-475a-8b5e-395688ec31d6"}
{"level":"error","ts":"2026-07-13T16:08:53.364Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"ef659015-9c54-475a-8b5e-395688ec31d6","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:53.813Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"57bb317e-88b7-46a0-bbbd-d796119da385"}
{"level":"info","ts":"2026-07-13T16:08:53.813Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"57bb317e-88b7-46a0-bbbd-d796119da385"}
{"level":"info","ts":"2026-07-13T16:08:53.813Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"57bb317e-88b7-46a0-bbbd-d796119da385","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:53.814Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"57bb317e-88b7-46a0-bbbd-d796119da385","Found 1 head Pod":"long-running-rfrb2-head-q8chr","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:137,Signal:0,Reason:Error,Message:,StartedAt:2026-07-13 16:08:22 +0000 UTC,FinishedAt:2026-07-13 16:08:53 +0000 UTC,ContainerID:cri-o://f2e550d73a84cba0972f84d2b7851fce3782797d88ba129d20df357ea7cb5873,}"}
{"level":"info","ts":"2026-07-13T16:08:53.814Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"57bb317e-88b7-46a0-bbbd-d796119da385","head Pod":"long-running-rfrb2-head-q8chr","shouldDelete":true,"reason":"The head Pod long-running-rfrb2-head-q8chr status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-07-13T16:08:53.818Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"57bb317e-88b7-46a0-bbbd-d796119da385","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod long-running-rfrb2-head-q8chr status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:53.818Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"57bb317e-88b7-46a0-bbbd-d796119da385","error":"The head Pod long-running-rfrb2-head-q8chr status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-07-13T16:08:53.818Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"57bb317e-88b7-46a0-bbbd-d796119da385","podName":"long-running-rfrb2-head-q8chr","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-13T16:08:53.818Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"57bb317e-88b7-46a0-bbbd-d796119da385","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:08:32Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:08:32Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}]}
{"level":"info","ts":"2026-07-13T16:08:53.818Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"57bb317e-88b7-46a0-bbbd-d796119da385","name":"long-running-rfrb2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:08:32Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:08:32Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:08:53.829Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"57bb317e-88b7-46a0-bbbd-d796119da385"}
{"level":"error","ts":"2026-07-13T16:08:53.829Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"57bb317e-88b7-46a0-bbbd-d796119da385","error":"The head Pod long-running-rfrb2-head-q8chr status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:53.830Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"0919a3ea-804c-4248-a388-f6b842f535b6","rayCluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:53.830Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"f0835446-5a50-4272-ab65-3d6354dca03a","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:53.830Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"0919a3ea-804c-4248-a388-f6b842f535b6","pod":"long-running-rfrb2-head-q8chr","ip":"10.133.0.38"}
{"level":"info","ts":"2026-07-13T16:08:53.830Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"0919a3ea-804c-4248-a388-f6b842f535b6"}
{"level":"info","ts":"2026-07-13T16:08:53.830Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"e446ba8c-66d3-4d49-a07d-e9ddaac396ac","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:53.830Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"e446ba8c-66d3-4d49-a07d-e9ddaac396ac","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-07-13 16:08:17 +0000 UTC","Now":"2026-07-13T16:08:53.830Z","ShutdownTime":"2026-07-13T16:18:17.000Z"}
{"level":"info","ts":"2026-07-13T16:08:53.830Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"e446ba8c-66d3-4d49-a07d-e9ddaac396ac","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T16:08:53.830Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"e446ba8c-66d3-4d49-a07d-e9ddaac396ac","seconds":565}
{"level":"info","ts":"2026-07-13T16:08:53.834Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"f0835446-5a50-4272-ab65-3d6354dca03a","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:54.300Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089"}
{"level":"info","ts":"2026-07-13T16:08:54.300Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089"}
{"level":"info","ts":"2026-07-13T16:08:54.300Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:54.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-13T16:08:54.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:08:54.301Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:54.301Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:08:54.301Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:08:54.301Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:08:54.301Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089"}
{"level":"info","ts":"2026-07-13T16:08:54.301Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089","nodeType":"worker","rayStartParams":{"address":"long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:08:54.301Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089","rayStartCmd":"ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:54.301Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:08:54.301Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089","enabled":true}
{"level":"error","ts":"2026-07-13T16:08:54.304Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:54.304Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:08:54.304Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:54.304Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:54.304Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:54.304Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"},"newHead":{"serviceName":"long-running-rfrb2-head-svc"}}
{"level":"info","ts":"2026-07-13T16:08:54.304Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089","name":"long-running-rfrb2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-rfrb2-head-q8chr","serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:08:32Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:08:32Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:08:54.318Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089"}
{"level":"error","ts":"2026-07-13T16:08:54.318Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"56f521b1-f5f3-4507-81a8-fd44d6210089","error":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:54.318Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"2ae31d7f-981a-4205-87eb-2362ccf1bd36","rayCluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:54.318Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"ddb44422-5ea1-4bb1-9843-909d12f7d2d1","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:54.318Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec"}
{"level":"info","ts":"2026-07-13T16:08:54.318Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec"}
{"level":"info","ts":"2026-07-13T16:08:54.318Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"b2f5ecc6-c1a5-4be3-9af6-cfc1363c00d9","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:54.318Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"b2f5ecc6-c1a5-4be3-9af6-cfc1363c00d9","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-07-13 16:08:17 +0000 UTC","Now":"2026-07-13T16:08:54.318Z","ShutdownTime":"2026-07-13T16:18:17.000Z"}
{"level":"info","ts":"2026-07-13T16:08:54.318Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"b2f5ecc6-c1a5-4be3-9af6-cfc1363c00d9","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T16:08:54.318Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"b2f5ecc6-c1a5-4be3-9af6-cfc1363c00d9","seconds":564}
{"level":"info","ts":"2026-07-13T16:08:54.318Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec","1 head service found":"long-running-rfrb2-head-svc"}
{"level":"info","ts":"2026-07-13T16:08:54.318Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec"}
{"level":"info","ts":"2026-07-13T16:08:54.318Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:08:54.318Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec"}
{"level":"info","ts":"2026-07-13T16:08:54.318Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec","mode":"IntegratedOAuth","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:54.318Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec","cluster":"long-running-rfrb2","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:08:54.318Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec","cluster":"long-running-rfrb2","authType":"IntegratedOAuth","serviceAccount":"long-running-rfrb2-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T16:08:54.318Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-rfrb2","ray.io/group":"headgroup","ray.io/identifier":"long-running-rfrb2-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:08:54.318Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T16:08:54.318Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T16:08:54.318Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T16:08:54.318Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec","enabled":true}
{"level":"error","ts":"2026-07-13T16:08:54.322Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-rfrb2-head-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:54.322Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec","error":"FailedCreateHeadPod\npods \"long-running-rfrb2-head-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:08:54.322Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:54.322Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:54.322Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec","filter labels":["test-ns-x5qvg",{"ray.io/cluster":"long-running-rfrb2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:08:54.322Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:08:32Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:08:32Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"long-running-rfrb2-head-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}]}
{"level":"info","ts":"2026-07-13T16:08:54.322Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec","name":"long-running-rfrb2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:08:32Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-rfrb2-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:08:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-rfrb2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:08:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:08:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:08:32Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"long-running-rfrb2-head-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:08:54.322Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"ddb44422-5ea1-4bb1-9843-909d12f7d2d1","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:54.333Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"2ae31d7f-981a-4205-87eb-2362ccf1bd36","certificate":"ray-head-cert-long-running-rfrb2","dnsNames":["long-running-rfrb2-head-svc","localhost","long-running-rfrb2-head-svc.test-ns-x5qvg.svc","long-running-rfrb2-head-svc.test-ns-x5qvg.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:08:54.336Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec"}
{"level":"error","ts":"2026-07-13T16:08:54.336Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"87d82890-21e0-458f-aa5e-77f560172cec","error":"FailedCreateHeadPod\npods \"long-running-rfrb2-head-\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:54.336Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"579bab1c-7d8a-4907-b1f1-f9bbdb58613d","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:54.336Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"ae2d1687-0db3-4e6a-b93f-26fc3530a321","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:54.336Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"ae2d1687-0db3-4e6a-b93f-26fc3530a321","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-07-13 16:08:17 +0000 UTC","Now":"2026-07-13T16:08:54.336Z","ShutdownTime":"2026-07-13T16:18:17.000Z"}
{"level":"info","ts":"2026-07-13T16:08:54.336Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"ae2d1687-0db3-4e6a-b93f-26fc3530a321","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-07-13T16:08:54.336Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"ae2d1687-0db3-4e6a-b93f-26fc3530a321","seconds":564}
{"level":"info","ts":"2026-07-13T16:08:54.340Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"579bab1c-7d8a-4907-b1f1-f9bbdb58613d","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:54.348Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"2ae31d7f-981a-4205-87eb-2362ccf1bd36","certificate":"ray-worker-cert-long-running-rfrb2","dnsNames":["long-running-rfrb2-worker-svc","localhost","long-running-rfrb2-small-group","long-running-rfrb2-worker-svc.test-ns-x5qvg.svc","long-running-rfrb2-worker-svc.test-ns-x5qvg.svc.cluster.local","*.long-running-rfrb2-worker-svc.test-ns-x5qvg.svc","*.long-running-rfrb2-worker-svc.test-ns-x5qvg.svc.cluster.local","*-worker-*.test-ns-x5qvg.svc","*-worker-*.test-ns-x5qvg.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:08:54.348Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"2ae31d7f-981a-4205-87eb-2362ccf1bd36"}
{"level":"info","ts":"2026-07-13T16:08:54.348Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"980d7d4b-ebac-48ac-ad5d-07f6f8c0cad6","rayCluster":"long-running-rfrb2"}
{"level":"error","ts":"2026-07-13T16:08:54.353Z","msg":"Failed to update certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"980d7d4b-ebac-48ac-ad5d-07f6f8c0cad6","error":"Operation cannot be fulfilled on certificates.cert-manager.io \"ray-worker-cert-long-running-rfrb2\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).updateCertificateWithPodIPs\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:465\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:212\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:08:54.353Z","msg":"Failed to update worker certificate with pod IPs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"980d7d4b-ebac-48ac-ad5d-07f6f8c0cad6","error":"Operation cannot be fulfilled on certificates.cert-manager.io \"ray-worker-cert-long-running-rfrb2\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:213\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:54.353Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"980d7d4b-ebac-48ac-ad5d-07f6f8c0cad6"}
{"level":"error","ts":"2026-07-13T16:08:54.353Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"980d7d4b-ebac-48ac-ad5d-07f6f8c0cad6","error":"Operation cannot be fulfilled on certificates.cert-manager.io \"ray-worker-cert-long-running-rfrb2\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:08:54.358Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"233535fa-fb71-4964-82cc-51801b550664","rayCluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:08:54.359Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"233535fa-fb71-4964-82cc-51801b550664"}
{"level":"info","ts":"2026-07-13T16:08:57.156Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"7c0eca6e-5482-4b74-b47c-1455be3813c6","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:57.156Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"7c0eca6e-5482-4b74-b47c-1455be3813c6","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:57.156Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"7c0eca6e-5482-4b74-b47c-1455be3813c6","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:08:57.166Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"7c0eca6e-5482-4b74-b47c-1455be3813c6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-5jsfr","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-07-13T16:08:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:08:47Z","endTime":"2026-07-13T16:08:48Z"},"jobId":"fail-5jsfr","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:08:47,093\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-5jsfr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:08:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:08:57.166Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"7c0eca6e-5482-4b74-b47c-1455be3813c6","old JobStatus":"","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-07-13T16:08:57.178Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"083745c2-6399-4754-9c30-7f37c505b3ea","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:57.178Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"083745c2-6399-4754-9c30-7f37c505b3ea","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:57.178Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"083745c2-6399-4754-9c30-7f37c505b3ea","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:08:57.180Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"083745c2-6399-4754-9c30-7f37c505b3ea","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:08:47Z","endTime":"2026-07-13T16:08:48Z"},"jobId":"fail-5jsfr","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:08:47,093\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-5jsfr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:08:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:08:47Z","endTime":"2026-07-13T16:08:48Z"},"jobId":"fail-5jsfr","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:08:47,093\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-5jsfr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:08:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:08:57.687Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"d4747055-bfcd-46b8-8590-e610e4353816","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:57.687Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"d4747055-bfcd-46b8-8590-e610e4353816","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:57.687Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"d4747055-bfcd-46b8-8590-e610e4353816","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:08:57.690Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"d4747055-bfcd-46b8-8590-e610e4353816","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:08:47Z","endTime":"2026-07-13T16:08:48Z"},"jobId":"fail-5jsfr","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:08:47,093\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-5jsfr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:08:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:08:47Z","endTime":"2026-07-13T16:08:48Z"},"jobId":"fail-5jsfr","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:08:47,093\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-5jsfr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:08:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:08:58.696Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"54de8d3b-0788-4e53-a353-c90c02fd2f9c","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:58.696Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"54de8d3b-0788-4e53-a353-c90c02fd2f9c","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:58.696Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"54de8d3b-0788-4e53-a353-c90c02fd2f9c","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:08:58.699Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"54de8d3b-0788-4e53-a353-c90c02fd2f9c","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:08:47Z","endTime":"2026-07-13T16:08:48Z"},"jobId":"fail-5jsfr","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:08:47,093\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-5jsfr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:08:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:08:47Z","endTime":"2026-07-13T16:08:48Z"},"jobId":"fail-5jsfr","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:08:47,093\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-5jsfr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:08:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:08:58.715Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"25b02be1-5853-4d0e-923d-ff6c2aa908f9","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:58.715Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"25b02be1-5853-4d0e-923d-ff6c2aa908f9","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:08:58.715Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"25b02be1-5853-4d0e-923d-ff6c2aa908f9","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:08:58.718Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"25b02be1-5853-4d0e-923d-ff6c2aa908f9","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:08:47Z","endTime":"2026-07-13T16:08:48Z"},"jobId":"fail-5jsfr","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:08:47,093\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-5jsfr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:08:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-07-13T16:08:47Z","endTime":"2026-07-13T16:08:48Z"},"jobId":"fail-5jsfr","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Failed","reason":"AppFailed","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-07-13 16:08:47,093\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-5jsfr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-07-13T16:08:39Z","succeeded":0,"failed":1,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-07-13T16:08:58.718Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"25b02be1-5853-4d0e-923d-ff6c2aa908f9","old JobStatus":"FAILED","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-07-13T16:08:58.730Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"9241655d-86bc-47df-a316-0f2e55954cd1","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:08:58.730Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"9241655d-86bc-47df-a316-0f2e55954cd1","ShutdownAfterJobFinishes":false,"ClusterSelector":{"ray.io/cluster":"raycluster"},"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 16:08:58 +0000 UTC","Now":"2026-07-13T16:08:58.730Z","ShutdownTime":"2026-07-13T16:08:58.000Z"}
{"level":"info","ts":"2026-07-13T16:09:00.010Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.010Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-9wtq2"}
{"level":"info","ts":"2026-07-13T16:09:00.021Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0363ea0d-c132-4e20-8b47-28ade1aff911","namespacedName":{"name":"ray-cluster","namespace":"test-ns-9wtq2"}}
{"level":"info","ts":"2026-07-13T16:09:00.021Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a4f1459a-845e-42fd-9fe2-f28588c96898","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.021Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a4f1459a-845e-42fd-9fe2-f28588c96898","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.021Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"57227bc6-9d3d-4e56-b9de-66a1eee9edc6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.021Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0363ea0d-c132-4e20-8b47-28ade1aff911","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.021Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0363ea0d-c132-4e20-8b47-28ade1aff911","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.021Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0363ea0d-c132-4e20-8b47-28ade1aff911","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.026Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.026Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-9wtq2"}
{"level":"info","ts":"2026-07-13T16:09:00.029Z","logger":"controllers.RayCluster","msg":"Created service account for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b5c73cf0-29e3-4663-bb0d-e432de2d6252","name":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.031Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a4f1459a-845e-42fd-9fe2-f28588c96898","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.031Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a4f1459a-845e-42fd-9fe2-f28588c96898","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.121Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"57227bc6-9d3d-4e56-b9de-66a1eee9edc6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.122Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"fc72f8bc-d845-4bc4-80ac-03c0d8860bed","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.122Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a4f1459a-845e-42fd-9fe2-f28588c96898","rayCluster":"ray-cluster","secret":"ca-secret"}
{"level":"info","ts":"2026-07-13T16:09:00.122Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a4f1459a-845e-42fd-9fe2-f28588c96898","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.128Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0363ea0d-c132-4e20-8b47-28ade1aff911","name":"ray-cluster-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:09:00.131Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a4f1459a-845e-42fd-9fe2-f28588c96898","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.132Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"fc72f8bc-d845-4bc4-80ac-03c0d8860bed","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:09:00.221Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"9764cfa4-23d8-42f5-bb91-91271458c28a","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:09:00.221Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"9764cfa4-23d8-42f5-bb91-91271458c28a","ShutdownAfterJobFinishes":false,"ClusterSelector":{"ray.io/cluster":"raycluster"},"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 16:08:58 +0000 UTC","Now":"2026-07-13T16:09:00.221Z","ShutdownTime":"2026-07-13T16:08:58.000Z"}
{"level":"info","ts":"2026-07-13T16:09:00.321Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a4f1459a-845e-42fd-9fe2-f28588c96898","rayCluster":"ray-cluster","podIPs":[]}
{"level":"info","ts":"2026-07-13T16:09:00.421Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding does not exist yet, will retry auth SA patch","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0363ea0d-c132-4e20-8b47-28ade1aff911","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.421Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a4f1459a-845e-42fd-9fe2-f28588c96898","rayCluster":"ray-cluster","podIPs":[]}
{"level":"error","ts":"2026-07-13T16:09:00.421Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a4f1459a-845e-42fd-9fe2-f28588c96898","error":"Certificate.cert-manager.io \"ray-worker-cert-ray-cluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:00.421Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a4f1459a-845e-42fd-9fe2-f28588c96898"}
{"level":"error","ts":"2026-07-13T16:09:00.421Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a4f1459a-845e-42fd-9fe2-f28588c96898","error":"Certificate.cert-manager.io \"ray-worker-cert-ray-cluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:00.421Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"fc72f8bc-d845-4bc4-80ac-03c0d8860bed","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:09:00.421Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"fc72f8bc-d845-4bc4-80ac-03c0d8860bed","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.421Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"04742097-999a-4954-b98d-394891d5c3df","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.421Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"c98ae9e9-88e2-47b4-9cbb-c594163503e6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.421Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"04742097-999a-4954-b98d-394891d5c3df"}
{"level":"info","ts":"2026-07-13T16:09:00.427Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"5e5402ea-fa38-4c21-80d9-aac46646069c","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.427Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"5e5402ea-fa38-4c21-80d9-aac46646069c"}
{"level":"error","ts":"2026-07-13T16:09:00.430Z","logger":"authentication-controller","msg":"Failed to update AuthenticationReady condition","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0363ea0d-c132-4e20-8b47-28ade1aff911","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleOIDCConfiguration\n\t/workspace/controllers/ray/authentication_controller.go:285\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:00.430Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding not ready yet, requeueing","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0363ea0d-c132-4e20-8b47-28ade1aff911","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.430Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"c89edede-bc96-4b0b-bf1c-87860de9b574","namespacedName":{"name":"ray-cluster","namespace":"test-ns-9wtq2"}}
{"level":"info","ts":"2026-07-13T16:09:00.430Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"c89edede-bc96-4b0b-bf1c-87860de9b574","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.430Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"c89edede-bc96-4b0b-bf1c-87860de9b574","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.430Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding does not exist yet, will retry auth SA patch","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"c89edede-bc96-4b0b-bf1c-87860de9b574","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.433Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"c98ae9e9-88e2-47b4-9cbb-c594163503e6","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:09:00.439Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding not ready yet, requeueing","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"c89edede-bc96-4b0b-bf1c-87860de9b574","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.439Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"fd137860-df45-4494-bdaa-6f3860a86577","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.439Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"fd137860-df45-4494-bdaa-6f3860a86577"}
{"level":"info","ts":"2026-07-13T16:09:00.521Z","logger":"controllers.RayCluster","msg":"Created role for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b5c73cf0-29e3-4663-bb0d-e432de2d6252","name":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.528Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"c98ae9e9-88e2-47b4-9cbb-c594163503e6","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:09:00.528Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"c98ae9e9-88e2-47b4-9cbb-c594163503e6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.528Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"c3fd6f33-8bed-4fa5-a379-bf4c8e208ca2","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.539Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"c3fd6f33-8bed-4fa5-a379-bf4c8e208ca2","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:09:00.548Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"c3fd6f33-8bed-4fa5-a379-bf4c8e208ca2","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:09:00.548Z","logger":"controllers.RayCluster","msg":"Created role binding for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b5c73cf0-29e3-4663-bb0d-e432de2d6252","name":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.548Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"c3fd6f33-8bed-4fa5-a379-bf4c8e208ca2","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.548Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b5c73cf0-29e3-4663-bb0d-e432de2d6252"}
{"level":"info","ts":"2026-07-13T16:09:00.548Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b5c73cf0-29e3-4663-bb0d-e432de2d6252"}
{"level":"info","ts":"2026-07-13T16:09:00.555Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0a6e06d8-5849-473c-a445-cbbd2efd8dc2","namespacedName":{"name":"ray-cluster","namespace":"test-ns-9wtq2"}}
{"level":"info","ts":"2026-07-13T16:09:00.555Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0a6e06d8-5849-473c-a445-cbbd2efd8dc2","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.555Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0a6e06d8-5849-473c-a445-cbbd2efd8dc2","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.555Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b5c73cf0-29e3-4663-bb0d-e432de2d6252","name":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:00.555Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b5c73cf0-29e3-4663-bb0d-e432de2d6252","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-13T16:09:00.555Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b5c73cf0-29e3-4663-bb0d-e432de2d6252","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:00.555Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b5c73cf0-29e3-4663-bb0d-e432de2d6252","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-13T16:09:00.555Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b5c73cf0-29e3-4663-bb0d-e432de2d6252","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:00.555Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b5c73cf0-29e3-4663-bb0d-e432de2d6252","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:00.555Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b5c73cf0-29e3-4663-bb0d-e432de2d6252","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:00.555Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b5c73cf0-29e3-4663-bb0d-e432de2d6252","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:09:00.555Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b5c73cf0-29e3-4663-bb0d-e432de2d6252","name":"ray-cluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:09:00.563Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b5c73cf0-29e3-4663-bb0d-e432de2d6252","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-9wtq2","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-13T16:09:00.563Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b5c73cf0-29e3-4663-bb0d-e432de2d6252"}
{"level":"error","ts":"2026-07-13T16:09:00.563Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b5c73cf0-29e3-4663-bb0d-e432de2d6252","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:00.563Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0eb20132-2b17-4777-bf28-faea82b02c66"}
{"level":"info","ts":"2026-07-13T16:09:00.563Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0eb20132-2b17-4777-bf28-faea82b02c66"}
{"level":"info","ts":"2026-07-13T16:09:00.563Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0eb20132-2b17-4777-bf28-faea82b02c66","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:00.563Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0eb20132-2b17-4777-bf28-faea82b02c66","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-13T16:09:00.563Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0eb20132-2b17-4777-bf28-faea82b02c66","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:00.564Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0eb20132-2b17-4777-bf28-faea82b02c66","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-13T16:09:00.564Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0eb20132-2b17-4777-bf28-faea82b02c66","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:00.564Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0eb20132-2b17-4777-bf28-faea82b02c66","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:00.564Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0eb20132-2b17-4777-bf28-faea82b02c66","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:00.564Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0eb20132-2b17-4777-bf28-faea82b02c66","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:09:00.564Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0eb20132-2b17-4777-bf28-faea82b02c66","name":"ray-cluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"}]},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:09:00.574Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0eb20132-2b17-4777-bf28-faea82b02c66"}
{"level":"error","ts":"2026-07-13T16:09:00.574Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0eb20132-2b17-4777-bf28-faea82b02c66","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:00.574Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1405a258-314b-498e-843b-1111b7e19e48"}
{"level":"info","ts":"2026-07-13T16:09:00.574Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1405a258-314b-498e-843b-1111b7e19e48"}
{"level":"info","ts":"2026-07-13T16:09:00.574Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1405a258-314b-498e-843b-1111b7e19e48","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:00.574Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1405a258-314b-498e-843b-1111b7e19e48","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-13T16:09:00.574Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1405a258-314b-498e-843b-1111b7e19e48","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:00.574Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1405a258-314b-498e-843b-1111b7e19e48","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-13T16:09:00.574Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1405a258-314b-498e-843b-1111b7e19e48","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:00.574Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1405a258-314b-498e-843b-1111b7e19e48","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:00.575Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1405a258-314b-498e-843b-1111b7e19e48","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:00.575Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1405a258-314b-498e-843b-1111b7e19e48","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:09:00.575Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1405a258-314b-498e-843b-1111b7e19e48","name":"ray-cluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"}]},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:09:00.575Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"54ae33d3-9e57-43c6-83b6-abdb77d38e07","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.575Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"63e18309-f641-4a6c-a883-3c43f57e2d56","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.575Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"54ae33d3-9e57-43c6-83b6-abdb77d38e07"}
{"level":"info","ts":"2026-07-13T16:09:00.621Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1405a258-314b-498e-843b-1111b7e19e48","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-9wtq2","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-13T16:09:00.621Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1405a258-314b-498e-843b-1111b7e19e48"}
{"level":"error","ts":"2026-07-13T16:09:00.621Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1405a258-314b-498e-843b-1111b7e19e48","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:00.621Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b0391f73-141b-406c-b637-a9d092a9eb86"}
{"level":"info","ts":"2026-07-13T16:09:00.621Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b0391f73-141b-406c-b637-a9d092a9eb86"}
{"level":"info","ts":"2026-07-13T16:09:00.621Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b0391f73-141b-406c-b637-a9d092a9eb86","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:00.621Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b0391f73-141b-406c-b637-a9d092a9eb86","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-13T16:09:00.621Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b0391f73-141b-406c-b637-a9d092a9eb86","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:00.621Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b0391f73-141b-406c-b637-a9d092a9eb86","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-13T16:09:00.621Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b0391f73-141b-406c-b637-a9d092a9eb86","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:00.621Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b0391f73-141b-406c-b637-a9d092a9eb86","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:00.621Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b0391f73-141b-406c-b637-a9d092a9eb86","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:00.621Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b0391f73-141b-406c-b637-a9d092a9eb86"}
{"level":"error","ts":"2026-07-13T16:09:00.621Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b0391f73-141b-406c-b637-a9d092a9eb86","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:00.630Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"63e18309-f641-4a6c-a883-3c43f57e2d56","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:09:00.639Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"63e18309-f641-4a6c-a883-3c43f57e2d56","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:09:00.639Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"63e18309-f641-4a6c-a883-3c43f57e2d56","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.641Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"580e73d7-b025-48ef-88d3-fdcfd1112ce4"}
{"level":"info","ts":"2026-07-13T16:09:00.641Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"580e73d7-b025-48ef-88d3-fdcfd1112ce4"}
{"level":"info","ts":"2026-07-13T16:09:00.641Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"580e73d7-b025-48ef-88d3-fdcfd1112ce4","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:00.641Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"580e73d7-b025-48ef-88d3-fdcfd1112ce4","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-13T16:09:00.641Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"580e73d7-b025-48ef-88d3-fdcfd1112ce4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:00.641Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"580e73d7-b025-48ef-88d3-fdcfd1112ce4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-13T16:09:00.641Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"580e73d7-b025-48ef-88d3-fdcfd1112ce4","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:00.641Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"580e73d7-b025-48ef-88d3-fdcfd1112ce4","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:00.641Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"580e73d7-b025-48ef-88d3-fdcfd1112ce4","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:00.641Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"580e73d7-b025-48ef-88d3-fdcfd1112ce4"}
{"level":"error","ts":"2026-07-13T16:09:00.641Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"580e73d7-b025-48ef-88d3-fdcfd1112ce4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:00.650Z","logger":"authentication-controller","msg":"Added auth proxy SA to autoscaler RoleBinding","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0a6e06d8-5849-473c-a445-cbbd2efd8dc2","roleBinding":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"authServiceAccount":"ray-cluster-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:09:00.654Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0a6e06d8-5849-473c-a445-cbbd2efd8dc2","name":"kuberay-gateway-access","namespace":"test-ns-9wtq2","operation":"created"}
{"level":"info","ts":"2026-07-13T16:09:00.663Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0a6e06d8-5849-473c-a445-cbbd2efd8dc2","name":"test-ns-9wtq2-ray-cluster","operation":"created"}
{"level":"info","ts":"2026-07-13T16:09:00.668Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0a6e06d8-5849-473c-a445-cbbd2efd8dc2","name":"kube-rbac-proxy-config-ray-cluster","operation":"created"}
{"level":"error","ts":"2026-07-13T16:09:00.674Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0a6e06d8-5849-473c-a445-cbbd2efd8dc2","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:00.674Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0a6e06d8-5849-473c-a445-cbbd2efd8dc2"}
{"level":"error","ts":"2026-07-13T16:09:00.674Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0a6e06d8-5849-473c-a445-cbbd2efd8dc2","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:00.679Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"80d84d4f-c226-414e-86d6-63555cdb59d4","namespacedName":{"name":"ray-cluster","namespace":"test-ns-9wtq2"}}
{"level":"info","ts":"2026-07-13T16:09:00.679Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"80d84d4f-c226-414e-86d6-63555cdb59d4","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.679Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"80d84d4f-c226-414e-86d6-63555cdb59d4","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.691Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"80d84d4f-c226-414e-86d6-63555cdb59d4","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.692Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"cb2ab46d-098d-4a59-b125-08f0ea35f99d","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.692Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"c5d8e0fc-d9f6-49e9-9a40-37d29888b5e5","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.692Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"c5d8e0fc-d9f6-49e9-9a40-37d29888b5e5"}
{"level":"info","ts":"2026-07-13T16:09:00.701Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"cb2ab46d-098d-4a59-b125-08f0ea35f99d","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:09:00.712Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"cb2ab46d-098d-4a59-b125-08f0ea35f99d","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:09:00.712Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"cb2ab46d-098d-4a59-b125-08f0ea35f99d","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:00.722Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"555e197e-bd33-4ff2-99ec-6f1bc6431122"}
{"level":"info","ts":"2026-07-13T16:09:00.722Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"555e197e-bd33-4ff2-99ec-6f1bc6431122"}
{"level":"info","ts":"2026-07-13T16:09:00.722Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"555e197e-bd33-4ff2-99ec-6f1bc6431122","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:00.722Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"555e197e-bd33-4ff2-99ec-6f1bc6431122","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-13T16:09:00.722Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"555e197e-bd33-4ff2-99ec-6f1bc6431122","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:00.722Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"555e197e-bd33-4ff2-99ec-6f1bc6431122","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-13T16:09:00.722Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"555e197e-bd33-4ff2-99ec-6f1bc6431122","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:00.722Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"555e197e-bd33-4ff2-99ec-6f1bc6431122","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:00.722Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"555e197e-bd33-4ff2-99ec-6f1bc6431122","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:00.722Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"555e197e-bd33-4ff2-99ec-6f1bc6431122"}
{"level":"error","ts":"2026-07-13T16:09:00.722Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"555e197e-bd33-4ff2-99ec-6f1bc6431122","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:00.883Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"e5aeb97d-4518-46b5-9525-5daec3b5e1c5"}
{"level":"info","ts":"2026-07-13T16:09:00.883Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"e5aeb97d-4518-46b5-9525-5daec3b5e1c5"}
{"level":"info","ts":"2026-07-13T16:09:00.883Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"e5aeb97d-4518-46b5-9525-5daec3b5e1c5","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:00.883Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"e5aeb97d-4518-46b5-9525-5daec3b5e1c5","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-13T16:09:00.883Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"e5aeb97d-4518-46b5-9525-5daec3b5e1c5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:00.883Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"e5aeb97d-4518-46b5-9525-5daec3b5e1c5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-13T16:09:00.883Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"e5aeb97d-4518-46b5-9525-5daec3b5e1c5","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:00.883Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"e5aeb97d-4518-46b5-9525-5daec3b5e1c5","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:00.883Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"e5aeb97d-4518-46b5-9525-5daec3b5e1c5","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:00.883Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"e5aeb97d-4518-46b5-9525-5daec3b5e1c5"}
{"level":"error","ts":"2026-07-13T16:09:00.883Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"e5aeb97d-4518-46b5-9525-5daec3b5e1c5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:01.203Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d5a3889-3acc-40de-8b33-38a0899b46c4"}
{"level":"info","ts":"2026-07-13T16:09:01.203Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d5a3889-3acc-40de-8b33-38a0899b46c4"}
{"level":"info","ts":"2026-07-13T16:09:01.203Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d5a3889-3acc-40de-8b33-38a0899b46c4","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:01.203Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d5a3889-3acc-40de-8b33-38a0899b46c4","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-13T16:09:01.203Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d5a3889-3acc-40de-8b33-38a0899b46c4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:01.203Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d5a3889-3acc-40de-8b33-38a0899b46c4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-13T16:09:01.204Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d5a3889-3acc-40de-8b33-38a0899b46c4","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:01.204Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d5a3889-3acc-40de-8b33-38a0899b46c4","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:01.204Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d5a3889-3acc-40de-8b33-38a0899b46c4","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:01.204Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d5a3889-3acc-40de-8b33-38a0899b46c4"}
{"level":"error","ts":"2026-07-13T16:09:01.204Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d5a3889-3acc-40de-8b33-38a0899b46c4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:01.844Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b"}
{"level":"info","ts":"2026-07-13T16:09:01.844Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b"}
{"level":"info","ts":"2026-07-13T16:09:01.844Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:01.844Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b"}
{"level":"info","ts":"2026-07-13T16:09:01.844Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:09:01.844Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b"}
{"level":"info","ts":"2026-07-13T16:09:01.844Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:01.844Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:09:01.844Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-13T16:09:01.844Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:09:01.844Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-13T16:09:01.844Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:09:01.844Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:09:01.844Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b","enabled":true}
{"level":"info","ts":"2026-07-13T16:09:01.860Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b","name":"ray-cluster-head-xmqjl"}
{"level":"info","ts":"2026-07-13T16:09:01.860Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:09:01.860Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:01.860Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:01.860Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:01.860Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:01.860Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:01.860Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:01.860Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:01.860Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"cc36d629-f946-4e52-9386-99948828f28b","seconds":300}
{"level":"info","ts":"2026-07-13T16:09:01.861Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4967b39b-da76-4418-8192-a6fa3b3f2f26"}
{"level":"info","ts":"2026-07-13T16:09:01.861Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4967b39b-da76-4418-8192-a6fa3b3f2f26"}
{"level":"info","ts":"2026-07-13T16:09:01.861Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4967b39b-da76-4418-8192-a6fa3b3f2f26","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:01.861Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4967b39b-da76-4418-8192-a6fa3b3f2f26","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:01.861Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4967b39b-da76-4418-8192-a6fa3b3f2f26","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:01.861Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4967b39b-da76-4418-8192-a6fa3b3f2f26","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:09:01.861Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4967b39b-da76-4418-8192-a6fa3b3f2f26","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:01.861Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4967b39b-da76-4418-8192-a6fa3b3f2f26","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:01.861Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4967b39b-da76-4418-8192-a6fa3b3f2f26","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:01.861Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4967b39b-da76-4418-8192-a6fa3b3f2f26","podName":"ray-cluster-head-xmqjl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:01.861Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4967b39b-da76-4418-8192-a6fa3b3f2f26","podName":"ray-cluster-head-xmqjl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:01.861Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4967b39b-da76-4418-8192-a6fa3b3f2f26","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"ray-cluster-head-svc"},"newHead":{"podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"}}
{"level":"info","ts":"2026-07-13T16:09:01.861Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4967b39b-da76-4418-8192-a6fa3b3f2f26","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:09:01.872Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f5eb86ce-9ab0-4f5e-805b-28ced01f9de0"}
{"level":"info","ts":"2026-07-13T16:09:01.872Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f5eb86ce-9ab0-4f5e-805b-28ced01f9de0"}
{"level":"info","ts":"2026-07-13T16:09:01.872Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f5eb86ce-9ab0-4f5e-805b-28ced01f9de0","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:01.872Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f5eb86ce-9ab0-4f5e-805b-28ced01f9de0","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:01.872Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f5eb86ce-9ab0-4f5e-805b-28ced01f9de0","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:01.872Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f5eb86ce-9ab0-4f5e-805b-28ced01f9de0","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:09:01.872Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f5eb86ce-9ab0-4f5e-805b-28ced01f9de0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:01.872Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f5eb86ce-9ab0-4f5e-805b-28ced01f9de0","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:01.872Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f5eb86ce-9ab0-4f5e-805b-28ced01f9de0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:01.872Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f5eb86ce-9ab0-4f5e-805b-28ced01f9de0","podName":"ray-cluster-head-xmqjl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:01.872Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f5eb86ce-9ab0-4f5e-805b-28ced01f9de0","podName":"ray-cluster-head-xmqjl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:01.872Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"94f40f87-46a1-4438-90c4-fef9f8b7914e","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:01.872Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"7166265c-a7d3-4ae0-8ce8-80763ec9b571","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:01.872Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f5eb86ce-9ab0-4f5e-805b-28ced01f9de0","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"ray-cluster-head-svc"},"newHead":{"podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"}}
{"level":"info","ts":"2026-07-13T16:09:01.872Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"94f40f87-46a1-4438-90c4-fef9f8b7914e"}
{"level":"info","ts":"2026-07-13T16:09:01.872Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f5eb86ce-9ab0-4f5e-805b-28ced01f9de0","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:09:01.878Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f5eb86ce-9ab0-4f5e-805b-28ced01f9de0","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-9wtq2","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-13T16:09:01.878Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f5eb86ce-9ab0-4f5e-805b-28ced01f9de0"}
{"level":"error","ts":"2026-07-13T16:09:01.878Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f5eb86ce-9ab0-4f5e-805b-28ced01f9de0","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:01.883Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"7166265c-a7d3-4ae0-8ce8-80763ec9b571","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:09:01.883Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9cf76311-9c3f-424f-a13b-b30a3b72301f"}
{"level":"info","ts":"2026-07-13T16:09:01.883Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9cf76311-9c3f-424f-a13b-b30a3b72301f"}
{"level":"info","ts":"2026-07-13T16:09:01.883Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9cf76311-9c3f-424f-a13b-b30a3b72301f","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:01.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9cf76311-9c3f-424f-a13b-b30a3b72301f","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:01.884Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9cf76311-9c3f-424f-a13b-b30a3b72301f","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:01.884Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9cf76311-9c3f-424f-a13b-b30a3b72301f","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:09:01.884Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9cf76311-9c3f-424f-a13b-b30a3b72301f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:01.884Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9cf76311-9c3f-424f-a13b-b30a3b72301f","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:01.884Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9cf76311-9c3f-424f-a13b-b30a3b72301f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:01.884Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9cf76311-9c3f-424f-a13b-b30a3b72301f","podName":"ray-cluster-head-xmqjl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:01.884Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9cf76311-9c3f-424f-a13b-b30a3b72301f","podName":"ray-cluster-head-xmqjl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:01.884Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9cf76311-9c3f-424f-a13b-b30a3b72301f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:01.884Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9cf76311-9c3f-424f-a13b-b30a3b72301f","seconds":300}
{"level":"info","ts":"2026-07-13T16:09:01.893Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"7166265c-a7d3-4ae0-8ce8-80763ec9b571","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:09:01.893Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"7166265c-a7d3-4ae0-8ce8-80763ec9b571","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:01.906Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8ffaf64e-b731-43ce-a272-1b77fd8b6f04"}
{"level":"info","ts":"2026-07-13T16:09:01.906Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8ffaf64e-b731-43ce-a272-1b77fd8b6f04"}
{"level":"info","ts":"2026-07-13T16:09:01.906Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8ffaf64e-b731-43ce-a272-1b77fd8b6f04","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:01.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8ffaf64e-b731-43ce-a272-1b77fd8b6f04","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:01.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8ffaf64e-b731-43ce-a272-1b77fd8b6f04","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:01.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8ffaf64e-b731-43ce-a272-1b77fd8b6f04","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:09:01.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8ffaf64e-b731-43ce-a272-1b77fd8b6f04","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:01.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8ffaf64e-b731-43ce-a272-1b77fd8b6f04","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:01.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8ffaf64e-b731-43ce-a272-1b77fd8b6f04","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:01.906Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8ffaf64e-b731-43ce-a272-1b77fd8b6f04","podName":"ray-cluster-head-xmqjl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:01.907Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8ffaf64e-b731-43ce-a272-1b77fd8b6f04","podName":"ray-cluster-head-xmqjl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:01.907Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8ffaf64e-b731-43ce-a272-1b77fd8b6f04","old conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"new conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}]}
{"level":"info","ts":"2026-07-13T16:09:01.907Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8ffaf64e-b731-43ce-a272-1b77fd8b6f04","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:09:01.931Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"279d5eca-d672-4780-8b00-96b392d8cb08","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:01.931Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"102cc0a4-06cd-41d6-a8f7-0137fe16a219","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:01.932Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"279d5eca-d672-4780-8b00-96b392d8cb08"}
{"level":"info","ts":"2026-07-13T16:09:01.942Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"102cc0a4-06cd-41d6-a8f7-0137fe16a219","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:09:01.953Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"102cc0a4-06cd-41d6-a8f7-0137fe16a219","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:09:01.953Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"102cc0a4-06cd-41d6-a8f7-0137fe16a219","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:02.318Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"dcc4002d-96b4-4408-b3e8-44f493a7461e"}
{"level":"info","ts":"2026-07-13T16:09:02.318Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"dcc4002d-96b4-4408-b3e8-44f493a7461e"}
{"level":"info","ts":"2026-07-13T16:09:02.318Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"dcc4002d-96b4-4408-b3e8-44f493a7461e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:02.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"dcc4002d-96b4-4408-b3e8-44f493a7461e","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:02.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"dcc4002d-96b4-4408-b3e8-44f493a7461e","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:02.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"dcc4002d-96b4-4408-b3e8-44f493a7461e","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:09:02.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"dcc4002d-96b4-4408-b3e8-44f493a7461e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:02.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"dcc4002d-96b4-4408-b3e8-44f493a7461e","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:02.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"dcc4002d-96b4-4408-b3e8-44f493a7461e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:02.318Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"dcc4002d-96b4-4408-b3e8-44f493a7461e","podName":"ray-cluster-head-xmqjl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:02.318Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"dcc4002d-96b4-4408-b3e8-44f493a7461e","podName":"ray-cluster-head-xmqjl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:02.318Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"dcc4002d-96b4-4408-b3e8-44f493a7461e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:02.318Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"dcc4002d-96b4-4408-b3e8-44f493a7461e","seconds":300}
{"level":"info","ts":"2026-07-13T16:09:02.395Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"161e23ad-e97d-4177-8bce-3ad81f60e0cd"}
{"level":"info","ts":"2026-07-13T16:09:02.395Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"161e23ad-e97d-4177-8bce-3ad81f60e0cd"}
{"level":"info","ts":"2026-07-13T16:09:02.395Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"161e23ad-e97d-4177-8bce-3ad81f60e0cd","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:02.395Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"161e23ad-e97d-4177-8bce-3ad81f60e0cd","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:02.395Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"161e23ad-e97d-4177-8bce-3ad81f60e0cd","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:02.395Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"161e23ad-e97d-4177-8bce-3ad81f60e0cd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:09:02.395Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"161e23ad-e97d-4177-8bce-3ad81f60e0cd","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:02.395Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"161e23ad-e97d-4177-8bce-3ad81f60e0cd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:02.395Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"161e23ad-e97d-4177-8bce-3ad81f60e0cd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:02.395Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"161e23ad-e97d-4177-8bce-3ad81f60e0cd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:02.395Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"161e23ad-e97d-4177-8bce-3ad81f60e0cd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:02.395Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"161e23ad-e97d-4177-8bce-3ad81f60e0cd","seconds":300}
{"level":"info","ts":"2026-07-13T16:09:02.400Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ab4672-5d05-4092-99fa-85deb51bd1ea"}
{"level":"info","ts":"2026-07-13T16:09:02.400Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ab4672-5d05-4092-99fa-85deb51bd1ea"}
{"level":"info","ts":"2026-07-13T16:09:02.400Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ab4672-5d05-4092-99fa-85deb51bd1ea","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:02.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ab4672-5d05-4092-99fa-85deb51bd1ea","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:02.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ab4672-5d05-4092-99fa-85deb51bd1ea","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:02.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ab4672-5d05-4092-99fa-85deb51bd1ea","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:09:02.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ab4672-5d05-4092-99fa-85deb51bd1ea","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:02.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ab4672-5d05-4092-99fa-85deb51bd1ea","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:02.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ab4672-5d05-4092-99fa-85deb51bd1ea","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:02.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ab4672-5d05-4092-99fa-85deb51bd1ea","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:02.400Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ab4672-5d05-4092-99fa-85deb51bd1ea","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:02.401Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ab4672-5d05-4092-99fa-85deb51bd1ea","seconds":300}
{"level":"info","ts":"2026-07-13T16:09:02.417Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a042b5d2-e65f-4d6a-ab68-c03fe49eeaf3"}
{"level":"info","ts":"2026-07-13T16:09:02.417Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a042b5d2-e65f-4d6a-ab68-c03fe49eeaf3"}
{"level":"info","ts":"2026-07-13T16:09:02.417Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a042b5d2-e65f-4d6a-ab68-c03fe49eeaf3","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:02.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a042b5d2-e65f-4d6a-ab68-c03fe49eeaf3","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:02.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a042b5d2-e65f-4d6a-ab68-c03fe49eeaf3","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:02.418Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a042b5d2-e65f-4d6a-ab68-c03fe49eeaf3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:09:02.418Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a042b5d2-e65f-4d6a-ab68-c03fe49eeaf3","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:02.418Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a042b5d2-e65f-4d6a-ab68-c03fe49eeaf3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:02.418Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a042b5d2-e65f-4d6a-ab68-c03fe49eeaf3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:02.418Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a042b5d2-e65f-4d6a-ab68-c03fe49eeaf3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:02.418Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a042b5d2-e65f-4d6a-ab68-c03fe49eeaf3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:02.418Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"a042b5d2-e65f-4d6a-ab68-c03fe49eeaf3","seconds":300}
{"level":"info","ts":"2026-07-13T16:09:02.420Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b6db5b7f-393e-44b3-97da-efeee2d895e7"}
{"level":"info","ts":"2026-07-13T16:09:02.420Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b6db5b7f-393e-44b3-97da-efeee2d895e7"}
{"level":"info","ts":"2026-07-13T16:09:02.420Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b6db5b7f-393e-44b3-97da-efeee2d895e7","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:02.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b6db5b7f-393e-44b3-97da-efeee2d895e7","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:02.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b6db5b7f-393e-44b3-97da-efeee2d895e7","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:02.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b6db5b7f-393e-44b3-97da-efeee2d895e7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:09:02.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b6db5b7f-393e-44b3-97da-efeee2d895e7","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-96vng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:02.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b6db5b7f-393e-44b3-97da-efeee2d895e7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:02.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b6db5b7f-393e-44b3-97da-efeee2d895e7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:02.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b6db5b7f-393e-44b3-97da-efeee2d895e7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:02.421Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b6db5b7f-393e-44b3-97da-efeee2d895e7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:02.421Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b6db5b7f-393e-44b3-97da-efeee2d895e7","seconds":300}
{"level":"info","ts":"2026-07-13T16:09:02.508Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"9c3b4630-11b8-44e9-a396-46519e424d68","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:09:02.514Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"9c3b4630-11b8-44e9-a396-46519e424d68","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:09:02.514Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"72d5a20a-debd-48df-b16f-af9b03e85dcd","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:09:02.518Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"72d5a20a-debd-48df-b16f-af9b03e85dcd","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:09:02.558Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"9a2db305-daa0-4b69-8264-252a3b8ed39e"}
{"level":"info","ts":"2026-07-13T16:09:02.564Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"fadf64a6-d7df-458a-9241-fc39379b17db","DeletionTimestamp":"2026-07-13 16:09:02 +0000 UTC"}
{"level":"info","ts":"2026-07-13T16:09:02.564Z","logger":"controllers.RayJob","msg":"Stop a ray job","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"fadf64a6-d7df-458a-9241-fc39379b17db","rayJob":"long-running-sfgvz"}
{"level":"error","ts":"2026-07-13T16:09:02.564Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"fadf64a6-d7df-458a-9241-fc39379b17db","error":"Post \"http:///api/jobs/long-running-sfgvz/stop\": http: no Host in request URL","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:123\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:02.564Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"fadf64a6-d7df-458a-9241-fc39379b17db","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T16:09:02.580Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"65e56a48-518a-496b-aa83-3d40cd3cd34e"}
{"level":"info","ts":"2026-07-13T16:09:02.580Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"26884926-a98a-47ba-9abb-64f3f797fffd","namespacedName":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:09:02.580Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"26884926-a98a-47ba-9abb-64f3f797fffd","mode":"IntegratedOAuth","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:09:02.580Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"26884926-a98a-47ba-9abb-64f3f797fffd","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:09:02.593Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"6a944fab-7289-454b-a73a-265fd5a7011e"}
{"level":"info","ts":"2026-07-13T16:09:02.593Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"cfcd5c42-c195-4b21-b0c0-4cde8c48d3ae","rayCluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:09:02.593Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"1da7a26b-bf31-4d72-8f1f-12ea22da2300"}
{"level":"info","ts":"2026-07-13T16:09:02.593Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"reconcileID":"d0782d38-9a99-4ef9-ad9b-900ee2019a83"}
{"level":"info","ts":"2026-07-13T16:09:02.593Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"cfcd5c42-c195-4b21-b0c0-4cde8c48d3ae"}
{"level":"error","ts":"2026-07-13T16:09:02.596Z","logger":"authentication-controller","msg":"Failed to update AuthenticationReady condition","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"26884926-a98a-47ba-9abb-64f3f797fffd","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-rfrb2\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleOIDCConfiguration\n\t/workspace/controllers/ray/authentication_controller.go:285\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:09:02.596Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"26884926-a98a-47ba-9abb-64f3f797fffd","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-rfrb2-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:02.596Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"26884926-a98a-47ba-9abb-64f3f797fffd"}
{"level":"error","ts":"2026-07-13T16:09:02.596Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"26884926-a98a-47ba-9abb-64f3f797fffd","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-rfrb2-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-x5qvg because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:02.596Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"26eaad6b-a7de-46a2-9b2e-48637edbaeb8","namespacedName":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:09:02.596Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"26eaad6b-a7de-46a2-9b2e-48637edbaeb8","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:09:02.596Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"26eaad6b-a7de-46a2-9b2e-48637edbaeb8","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:09:02.596Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"26eaad6b-a7de-46a2-9b2e-48637edbaeb8","namespace":"test-ns-x5qvg","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-13T16:09:02.596Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"26eaad6b-a7de-46a2-9b2e-48637edbaeb8","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-13T16:09:02.601Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"26eaad6b-a7de-46a2-9b2e-48637edbaeb8","name":"test-ns-x5qvg-long-running-rfrb2","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-13T16:09:02.606Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"26eaad6b-a7de-46a2-9b2e-48637edbaeb8","configMap":"kube-rbac-proxy-config-long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:09:02.614Z","logger":"raycluster-resource","msg":"default","name":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:09:02.614Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:09:02.622Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"26eaad6b-a7de-46a2-9b2e-48637edbaeb8","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:09:02.623Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"5259fc2d-8c55-46dc-8841-a27390490a0a","namespacedName":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:09:02.623Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"5259fc2d-8c55-46dc-8841-a27390490a0a","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:09:02.623Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"5259fc2d-8c55-46dc-8841-a27390490a0a","cluster":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:09:02.623Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"5259fc2d-8c55-46dc-8841-a27390490a0a","namespace":"test-ns-x5qvg","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-13T16:09:02.623Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"5259fc2d-8c55-46dc-8841-a27390490a0a","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-13T16:09:02.623Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"a8ce777f-cb9c-4711-bd93-eec991af3b29"}
{"level":"info","ts":"2026-07-13T16:09:02.623Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"bf1782ad-ce4d-4752-95d5-4b28ff63a58b","namespace":"test-ns-x5qvg","clusterName":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:09:02.623Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"a55f177c-f728-4050-ab2c-c4964939ecc1"}
{"level":"error","ts":"2026-07-13T16:09:02.626Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"5259fc2d-8c55-46dc-8841-a27390490a0a","error":"rayclusters.ray.io \"long-running-rfrb2\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:02.627Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"5259fc2d-8c55-46dc-8841-a27390490a0a"}
{"level":"error","ts":"2026-07-13T16:09:02.627Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"5259fc2d-8c55-46dc-8841-a27390490a0a","error":"rayclusters.ray.io \"long-running-rfrb2\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:02.627Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"83e97f70-e56a-46bf-8eda-626bff01e35f","namespacedName":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:09:02.627Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"83e97f70-e56a-46bf-8eda-626bff01e35f"}
{"level":"info","ts":"2026-07-13T16:09:02.627Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"83e97f70-e56a-46bf-8eda-626bff01e35f","namespace":"test-ns-x5qvg","authClusters":0}
{"level":"info","ts":"2026-07-13T16:09:02.627Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"83e97f70-e56a-46bf-8eda-626bff01e35f","namespace":"test-ns-x5qvg"}
{"level":"info","ts":"2026-07-13T16:09:02.632Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"f1b15aeb-6e57-4375-a3b6-8d447552e761","namespacedName":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"}}
{"level":"info","ts":"2026-07-13T16:09:02.632Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"f1b15aeb-6e57-4375-a3b6-8d447552e761"}
{"level":"info","ts":"2026-07-13T16:09:02.632Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"f1b15aeb-6e57-4375-a3b6-8d447552e761","namespace":"test-ns-x5qvg","authClusters":0}
{"level":"info","ts":"2026-07-13T16:09:02.632Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"f1b15aeb-6e57-4375-a3b6-8d447552e761","namespace":"test-ns-x5qvg"}
{"level":"error","ts":"2026-07-13T16:09:02.633Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"bf1782ad-ce4d-4752-95d5-4b28ff63a58b","issuer":"ray-selfsigned-issuer-long-running-rfrb2","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-rfrb2\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:09:02.636Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"bf1782ad-ce4d-4752-95d5-4b28ff63a58b","certificate":"ray-head-cert-long-running-rfrb2","error":"certificates.cert-manager.io \"ray-head-cert-long-running-rfrb2\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:02.649Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"bf1782ad-ce4d-4752-95d5-4b28ff63a58b","errorCount":2}
{"level":"info","ts":"2026-07-13T16:09:02.649Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"bf1782ad-ce4d-4752-95d5-4b28ff63a58b"}
{"level":"error","ts":"2026-07-13T16:09:02.649Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"bf1782ad-ce4d-4752-95d5-4b28ff63a58b","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-rfrb2\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:02.654Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"908037a3-8193-4978-8697-2ee326ebacdb","namespace":"test-ns-x5qvg","clusterName":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:09:02.654Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-rfrb2","namespace":"test-ns-x5qvg"},"namespace":"test-ns-x5qvg","name":"long-running-rfrb2","reconcileID":"908037a3-8193-4978-8697-2ee326ebacdb","namespace":"test-ns-x5qvg","clusterName":"long-running-rfrb2"}
{"level":"info","ts":"2026-07-13T16:09:02.951Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff1114c1-ed68-4a70-b4fc-1e7b52aab24a"}
{"level":"info","ts":"2026-07-13T16:09:02.951Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff1114c1-ed68-4a70-b4fc-1e7b52aab24a"}
{"level":"info","ts":"2026-07-13T16:09:02.951Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff1114c1-ed68-4a70-b4fc-1e7b52aab24a","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:02.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff1114c1-ed68-4a70-b4fc-1e7b52aab24a","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:02.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff1114c1-ed68-4a70-b4fc-1e7b52aab24a","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:02.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff1114c1-ed68-4a70-b4fc-1e7b52aab24a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:09:02.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff1114c1-ed68-4a70-b4fc-1e7b52aab24a","worker Pod":"raycluster-small-group-worker-96vng","shouldDelete":true,"reason":"The worker Pod raycluster-small-group-worker-96vng status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-07-13T16:09:02.955Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff1114c1-ed68-4a70-b4fc-1e7b52aab24a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"delete 1 unhealthy worker Pods","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:02.955Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff1114c1-ed68-4a70-b4fc-1e7b52aab24a","error":"delete 1 unhealthy worker Pods"}
{"level":"info","ts":"2026-07-13T16:09:02.955Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff1114c1-ed68-4a70-b4fc-1e7b52aab24a","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-07-13T16:09:02.955Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff1114c1-ed68-4a70-b4fc-1e7b52aab24a","name":"raycluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T15:49:35Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:02Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:09:02.967Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff1114c1-ed68-4a70-b4fc-1e7b52aab24a"}
{"level":"error","ts":"2026-07-13T16:09:02.967Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff1114c1-ed68-4a70-b4fc-1e7b52aab24a","error":"delete 1 unhealthy worker Pods","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:02.968Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"04cc2822-db6f-4f13-933c-8d8af0eadfb7","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:02.968Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"e603cd91-2a10-490f-a252-d1b7f5de0a49","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:02.968Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"04cc2822-db6f-4f13-933c-8d8af0eadfb7","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T16:09:02.968Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"04cc2822-db6f-4f13-933c-8d8af0eadfb7","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T16:09:02.968Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"04cc2822-db6f-4f13-933c-8d8af0eadfb7","rayCluster":"raycluster","podIPs":["10.133.0.18","10.134.0.25"]}
{"level":"info","ts":"2026-07-13T16:09:02.973Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"e603cd91-2a10-490f-a252-d1b7f5de0a49","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:02.973Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"4dcd6a54-210d-4624-b8b0-7ec814baf813"}
{"level":"info","ts":"2026-07-13T16:09:02.973Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"4dcd6a54-210d-4624-b8b0-7ec814baf813"}
{"level":"info","ts":"2026-07-13T16:09:02.973Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"4dcd6a54-210d-4624-b8b0-7ec814baf813","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:02.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"4dcd6a54-210d-4624-b8b0-7ec814baf813","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:02.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"4dcd6a54-210d-4624-b8b0-7ec814baf813","head Pod":"raycluster-head-fdvcn","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-fdvcn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:02.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"4dcd6a54-210d-4624-b8b0-7ec814baf813","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:09:02.973Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"4dcd6a54-210d-4624-b8b0-7ec814baf813","podName":"raycluster-small-group-worker-96vng","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-13T16:09:02.973Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"4dcd6a54-210d-4624-b8b0-7ec814baf813","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:02.973Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"4dcd6a54-210d-4624-b8b0-7ec814baf813","seconds":300}
{"level":"info","ts":"2026-07-13T16:09:03.166Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"51c6c494-2fda-4cc9-908b-b0316c4f41f1"}
{"level":"info","ts":"2026-07-13T16:09:03.166Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"51c6c494-2fda-4cc9-908b-b0316c4f41f1"}
{"level":"info","ts":"2026-07-13T16:09:03.166Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"51c6c494-2fda-4cc9-908b-b0316c4f41f1","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:03.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"51c6c494-2fda-4cc9-908b-b0316c4f41f1","Found 1 head Pod":"raycluster-head-fdvcn","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-07-13 15:49:13 +0000 UTC,FinishedAt:2026-07-13 16:09:02 +0000 UTC,ContainerID:cri-o://8f373871d7d0dc1e5e51910b0ad95be047d354accee4e71f7cde6b7d071339a3,}"}
{"level":"info","ts":"2026-07-13T16:09:03.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"51c6c494-2fda-4cc9-908b-b0316c4f41f1","head Pod":"raycluster-head-fdvcn","shouldDelete":true,"reason":"The head Pod raycluster-head-fdvcn status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-07-13T16:09:03.169Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"51c6c494-2fda-4cc9-908b-b0316c4f41f1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod raycluster-head-fdvcn status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:03.169Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"51c6c494-2fda-4cc9-908b-b0316c4f41f1","error":"The head Pod raycluster-head-fdvcn status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-07-13T16:09:03.169Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"51c6c494-2fda-4cc9-908b-b0316c4f41f1","old conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-07-13T16:09:03.169Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"51c6c494-2fda-4cc9-908b-b0316c4f41f1","name":"raycluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:02Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T15:49:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:03Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:09:03.180Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"51c6c494-2fda-4cc9-908b-b0316c4f41f1"}
{"level":"error","ts":"2026-07-13T16:09:03.180Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"51c6c494-2fda-4cc9-908b-b0316c4f41f1","error":"The head Pod raycluster-head-fdvcn status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:03.181Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"d84a0529-112d-4ae8-b755-6901e2107fa7","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:03.181Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"997dc647-7526-4855-b984-3c32c17fd28d","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:03.181Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"997dc647-7526-4855-b984-3c32c17fd28d","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"info","ts":"2026-07-13T16:09:03.181Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"997dc647-7526-4855-b984-3c32c17fd28d","pod":"raycluster-small-group-worker-96vng","ip":"10.133.0.18"}
{"level":"info","ts":"2026-07-13T16:09:03.181Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"997dc647-7526-4855-b984-3c32c17fd28d","rayCluster":"raycluster","podIPs":["10.134.0.25","10.133.0.18"]}
{"level":"info","ts":"2026-07-13T16:09:03.185Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"d84a0529-112d-4ae8-b755-6901e2107fa7","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:03.186Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fa05865c-e30c-4c05-89eb-08012f9c06e8"}
{"level":"info","ts":"2026-07-13T16:09:03.186Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fa05865c-e30c-4c05-89eb-08012f9c06e8"}
{"level":"info","ts":"2026-07-13T16:09:03.186Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fa05865c-e30c-4c05-89eb-08012f9c06e8","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:03.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fa05865c-e30c-4c05-89eb-08012f9c06e8","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-13T16:09:03.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fa05865c-e30c-4c05-89eb-08012f9c06e8","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:09:03.186Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fa05865c-e30c-4c05-89eb-08012f9c06e8","podName":"raycluster-head-fdvcn","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-13T16:09:03.186Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fa05865c-e30c-4c05-89eb-08012f9c06e8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:03.186Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"fa05865c-e30c-4c05-89eb-08012f9c06e8","seconds":300}
{"level":"info","ts":"2026-07-13T16:09:03.340Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2f327864-cf63-49d7-8183-aa7303a8a903"}
{"level":"info","ts":"2026-07-13T16:09:03.340Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2f327864-cf63-49d7-8183-aa7303a8a903"}
{"level":"info","ts":"2026-07-13T16:09:03.340Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2f327864-cf63-49d7-8183-aa7303a8a903","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:03.340Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2f327864-cf63-49d7-8183-aa7303a8a903","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:03.340Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2f327864-cf63-49d7-8183-aa7303a8a903","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:03.340Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2f327864-cf63-49d7-8183-aa7303a8a903","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:09:03.340Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2f327864-cf63-49d7-8183-aa7303a8a903","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:03.340Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2f327864-cf63-49d7-8183-aa7303a8a903","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:03.340Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2f327864-cf63-49d7-8183-aa7303a8a903","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:03.340Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2f327864-cf63-49d7-8183-aa7303a8a903","podName":"ray-cluster-head-xmqjl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:03.340Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2f327864-cf63-49d7-8183-aa7303a8a903","podName":"ray-cluster-head-xmqjl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:03.340Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2f327864-cf63-49d7-8183-aa7303a8a903","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"newHead":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"}}
{"level":"info","ts":"2026-07-13T16:09:03.340Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2f327864-cf63-49d7-8183-aa7303a8a903","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:09:03.352Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"494ccaad-d3ce-43b4-9308-1b2a200d63a0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:03.352Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"c590c487-31ec-45c3-8997-a0b6a81f168e","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:03.352Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"c590c487-31ec-45c3-8997-a0b6a81f168e","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:09:03.355Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"cf16360e-7bdd-4912-9691-6718d59e4ad7"}
{"level":"info","ts":"2026-07-13T16:09:03.355Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"cf16360e-7bdd-4912-9691-6718d59e4ad7"}
{"level":"info","ts":"2026-07-13T16:09:03.355Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"cf16360e-7bdd-4912-9691-6718d59e4ad7","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:03.355Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"cf16360e-7bdd-4912-9691-6718d59e4ad7","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-13T16:09:03.355Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"cf16360e-7bdd-4912-9691-6718d59e4ad7","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:09:03.355Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"cf16360e-7bdd-4912-9691-6718d59e4ad7","podName":"raycluster-head-fdvcn","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-13T16:09:03.356Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"cf16360e-7bdd-4912-9691-6718d59e4ad7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:03.356Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"cf16360e-7bdd-4912-9691-6718d59e4ad7","seconds":300}
{"level":"info","ts":"2026-07-13T16:09:03.422Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"c590c487-31ec-45c3-8997-a0b6a81f168e","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-9wtq2.svc","ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local"],"ipAddresses":["10.133.0.40","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:09:03.423Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"42e9f64d-6dc6-4b43-b219-6f82b48c8d1f"}
{"level":"info","ts":"2026-07-13T16:09:03.423Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"42e9f64d-6dc6-4b43-b219-6f82b48c8d1f"}
{"level":"info","ts":"2026-07-13T16:09:03.423Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"42e9f64d-6dc6-4b43-b219-6f82b48c8d1f","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:03.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"42e9f64d-6dc6-4b43-b219-6f82b48c8d1f","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-13T16:09:03.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"42e9f64d-6dc6-4b43-b219-6f82b48c8d1f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:09:03.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"42e9f64d-6dc6-4b43-b219-6f82b48c8d1f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:03.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"42e9f64d-6dc6-4b43-b219-6f82b48c8d1f","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:09:03.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"42e9f64d-6dc6-4b43-b219-6f82b48c8d1f","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:03.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"42e9f64d-6dc6-4b43-b219-6f82b48c8d1f","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:09:03.423Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"42e9f64d-6dc6-4b43-b219-6f82b48c8d1f"}
{"level":"info","ts":"2026-07-13T16:09:03.423Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"42e9f64d-6dc6-4b43-b219-6f82b48c8d1f","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:09:03.423Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"42e9f64d-6dc6-4b43-b219-6f82b48c8d1f","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:09:03.423Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"42e9f64d-6dc6-4b43-b219-6f82b48c8d1f","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:09:03.423Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"42e9f64d-6dc6-4b43-b219-6f82b48c8d1f","enabled":true}
{"level":"info","ts":"2026-07-13T16:09:03.426Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"494ccaad-d3ce-43b4-9308-1b2a200d63a0","name":"ray-cluster-head"}
{"level":"error","ts":"2026-07-13T16:09:03.426Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"42e9f64d-6dc6-4b43-b219-6f82b48c8d1f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:03.426Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"42e9f64d-6dc6-4b43-b219-6f82b48c8d1f","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:09:03.427Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"42e9f64d-6dc6-4b43-b219-6f82b48c8d1f","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated"}]}
{"level":"info","ts":"2026-07-13T16:09:03.427Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"42e9f64d-6dc6-4b43-b219-6f82b48c8d1f","name":"raycluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:03Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:03Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:09:03.521Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"494ccaad-d3ce-43b4-9308-1b2a200d63a0","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:09:03.521Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"494ccaad-d3ce-43b4-9308-1b2a200d63a0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:03.522Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"42e9f64d-6dc6-4b43-b219-6f82b48c8d1f"}
{"level":"error","ts":"2026-07-13T16:09:03.522Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"42e9f64d-6dc6-4b43-b219-6f82b48c8d1f","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:03.522Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8f9d584c-6f31-4acf-babc-f7cb3d30f905","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:03.621Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8f9d584c-6f31-4acf-babc-f7cb3d30f905","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:03.621Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d02fe443-aae8-4637-a98a-cf88a91a463f"}
{"level":"info","ts":"2026-07-13T16:09:03.621Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d02fe443-aae8-4637-a98a-cf88a91a463f"}
{"level":"info","ts":"2026-07-13T16:09:03.621Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d02fe443-aae8-4637-a98a-cf88a91a463f","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:03.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d02fe443-aae8-4637-a98a-cf88a91a463f","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-13T16:09:03.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d02fe443-aae8-4637-a98a-cf88a91a463f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:09:03.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d02fe443-aae8-4637-a98a-cf88a91a463f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:03.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d02fe443-aae8-4637-a98a-cf88a91a463f","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:09:03.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d02fe443-aae8-4637-a98a-cf88a91a463f","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:03.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d02fe443-aae8-4637-a98a-cf88a91a463f","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:09:03.622Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d02fe443-aae8-4637-a98a-cf88a91a463f"}
{"level":"info","ts":"2026-07-13T16:09:03.622Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"c590c487-31ec-45c3-8997-a0b6a81f168e","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-9wtq2.svc","ray-cluster-worker-svc.test-ns-9wtq2.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-9wtq2.svc","*.ray-cluster-worker-svc.test-ns-9wtq2.svc.cluster.local","*-worker-*.test-ns-9wtq2.svc","*-worker-*.test-ns-9wtq2.svc.cluster.local"],"ipAddresses":["10.133.0.40","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:09:03.622Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d02fe443-aae8-4637-a98a-cf88a91a463f","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:09:03.622Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"c590c487-31ec-45c3-8997-a0b6a81f168e"}
{"level":"info","ts":"2026-07-13T16:09:03.622Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d02fe443-aae8-4637-a98a-cf88a91a463f","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:09:03.622Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d02fe443-aae8-4637-a98a-cf88a91a463f","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:09:03.622Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d02fe443-aae8-4637-a98a-cf88a91a463f","enabled":true}
{"level":"info","ts":"2026-07-13T16:09:03.622Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"705ceb60-3bd6-4c4a-8466-03290a335697","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:03.622Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"705ceb60-3bd6-4c4a-8466-03290a335697","pod":"raycluster-head-fdvcn","ip":"10.134.0.25"}
{"level":"error","ts":"2026-07-13T16:09:03.625Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d02fe443-aae8-4637-a98a-cf88a91a463f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:03.625Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d02fe443-aae8-4637-a98a-cf88a91a463f","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:09:03.625Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d02fe443-aae8-4637-a98a-cf88a91a463f"}
{"level":"error","ts":"2026-07-13T16:09:03.625Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"d02fe443-aae8-4637-a98a-cf88a91a463f","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:03.633Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"705ceb60-3bd6-4c4a-8466-03290a335697","certificate":"ray-head-cert-raycluster","dnsNames":["raycluster-head-svc","localhost","raycluster-head-svc.test-ns-qbjjb.svc","raycluster-head-svc.test-ns-qbjjb.svc.cluster.local"],"ipAddresses":["10.134.0.25","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:09:03.636Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"5a21ea5d-1153-4150-bfaf-c10a17eb4117"}
{"level":"info","ts":"2026-07-13T16:09:03.636Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"5a21ea5d-1153-4150-bfaf-c10a17eb4117"}
{"level":"info","ts":"2026-07-13T16:09:03.636Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"5a21ea5d-1153-4150-bfaf-c10a17eb4117","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:03.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"5a21ea5d-1153-4150-bfaf-c10a17eb4117","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-13T16:09:03.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"5a21ea5d-1153-4150-bfaf-c10a17eb4117","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:09:03.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"5a21ea5d-1153-4150-bfaf-c10a17eb4117","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:03.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"5a21ea5d-1153-4150-bfaf-c10a17eb4117","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:09:03.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"5a21ea5d-1153-4150-bfaf-c10a17eb4117","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:03.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"5a21ea5d-1153-4150-bfaf-c10a17eb4117","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:09:03.636Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"5a21ea5d-1153-4150-bfaf-c10a17eb4117"}
{"level":"info","ts":"2026-07-13T16:09:03.636Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"5a21ea5d-1153-4150-bfaf-c10a17eb4117","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:09:03.636Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"5a21ea5d-1153-4150-bfaf-c10a17eb4117","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:09:03.636Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"5a21ea5d-1153-4150-bfaf-c10a17eb4117","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:09:03.636Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"5a21ea5d-1153-4150-bfaf-c10a17eb4117","enabled":true}
{"level":"error","ts":"2026-07-13T16:09:03.640Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"5a21ea5d-1153-4150-bfaf-c10a17eb4117","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:03.640Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"5a21ea5d-1153-4150-bfaf-c10a17eb4117","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:09:03.640Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"5a21ea5d-1153-4150-bfaf-c10a17eb4117"}
{"level":"error","ts":"2026-07-13T16:09:03.640Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"5a21ea5d-1153-4150-bfaf-c10a17eb4117","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:03.646Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"705ceb60-3bd6-4c4a-8466-03290a335697","certificate":"ray-worker-cert-raycluster","dnsNames":["raycluster-worker-svc","localhost","raycluster-small-group","raycluster-worker-svc.test-ns-qbjjb.svc","raycluster-worker-svc.test-ns-qbjjb.svc.cluster.local","*.raycluster-worker-svc.test-ns-qbjjb.svc","*.raycluster-worker-svc.test-ns-qbjjb.svc.cluster.local","*-worker-*.test-ns-qbjjb.svc","*-worker-*.test-ns-qbjjb.svc.cluster.local"],"ipAddresses":["10.134.0.25","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:09:03.646Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"705ceb60-3bd6-4c4a-8466-03290a335697","rayCluster":"raycluster","podIPs":["10.134.0.25"]}
{"level":"info","ts":"2026-07-13T16:09:03.660Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ee8170-7e0c-40f8-9f5d-4efc6b14c730"}
{"level":"info","ts":"2026-07-13T16:09:03.660Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ee8170-7e0c-40f8-9f5d-4efc6b14c730"}
{"level":"info","ts":"2026-07-13T16:09:03.661Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ee8170-7e0c-40f8-9f5d-4efc6b14c730","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:03.661Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ee8170-7e0c-40f8-9f5d-4efc6b14c730","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-13T16:09:03.661Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ee8170-7e0c-40f8-9f5d-4efc6b14c730","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:09:03.661Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ee8170-7e0c-40f8-9f5d-4efc6b14c730","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:03.661Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ee8170-7e0c-40f8-9f5d-4efc6b14c730","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:09:03.661Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ee8170-7e0c-40f8-9f5d-4efc6b14c730","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:03.661Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ee8170-7e0c-40f8-9f5d-4efc6b14c730","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:09:03.661Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ee8170-7e0c-40f8-9f5d-4efc6b14c730"}
{"level":"info","ts":"2026-07-13T16:09:03.661Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ee8170-7e0c-40f8-9f5d-4efc6b14c730","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:09:03.661Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ee8170-7e0c-40f8-9f5d-4efc6b14c730","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:09:03.661Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ee8170-7e0c-40f8-9f5d-4efc6b14c730","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:09:03.661Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ee8170-7e0c-40f8-9f5d-4efc6b14c730","enabled":true}
{"level":"error","ts":"2026-07-13T16:09:03.664Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ee8170-7e0c-40f8-9f5d-4efc6b14c730","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:03.664Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ee8170-7e0c-40f8-9f5d-4efc6b14c730","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:09:03.664Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ee8170-7e0c-40f8-9f5d-4efc6b14c730"}
{"level":"error","ts":"2026-07-13T16:09:03.664Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"90ee8170-7e0c-40f8-9f5d-4efc6b14c730","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:03.721Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb"}
{"level":"info","ts":"2026-07-13T16:09:03.721Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb"}
{"level":"info","ts":"2026-07-13T16:09:03.721Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:03.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-13T16:09:03.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-07-13T16:09:03.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:03.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:09:03.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:03.721Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:09:03.721Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb"}
{"level":"info","ts":"2026-07-13T16:09:03.721Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:09:03.721Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:09:03.721Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:09:03.721Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb","enabled":true}
{"level":"error","ts":"2026-07-13T16:09:03.724Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:03.724Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:09:03.724Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:03.724Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:03.724Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:03.724Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"newHead":{"serviceName":"raycluster-head-svc"}}
{"level":"info","ts":"2026-07-13T16:09:03.724Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb","name":"raycluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:03Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:03Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:09:03.735Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb"}
{"level":"error","ts":"2026-07-13T16:09:03.735Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b797322b-531e-4cd5-b884-8dfc5ef674bb","error":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:03.735Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22"}
{"level":"info","ts":"2026-07-13T16:09:03.735Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22"}
{"level":"info","ts":"2026-07-13T16:09:03.736Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:03.736Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22"}
{"level":"info","ts":"2026-07-13T16:09:03.736Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:09:03.736Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22"}
{"level":"info","ts":"2026-07-13T16:09:03.736Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:03.736Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22","cluster":"raycluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:09:03.736Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22","cluster":"raycluster","authType":"IntegratedOAuth","serviceAccount":"raycluster-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T16:09:03.736Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster","ray.io/group":"headgroup","ray.io/identifier":"raycluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:09:03.736Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"f88c7df0-ed2d-45cf-87de-e7121bf7c0ee","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:03.736Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"73cedcfa-c3d0-47e4-8fb4-e855e299b2fa","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:03.736Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T16:09:03.736Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T16:09:03.736Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T16:09:03.736Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22","enabled":true}
{"level":"error","ts":"2026-07-13T16:09:03.739Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:03.739Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:09:03.739Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:03.739Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:03.739Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:03.739Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"newHead":{"serviceName":"raycluster-head-svc"}}
{"level":"info","ts":"2026-07-13T16:09:03.739Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22","name":"raycluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:03Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.25","serviceIP":"10.134.0.25","podName":"raycluster-head-fdvcn","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:03Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:09:03.740Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"73cedcfa-c3d0-47e4-8fb4-e855e299b2fa","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:03.745Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"f88c7df0-ed2d-45cf-87de-e7121bf7c0ee","certificate":"ray-head-cert-raycluster","dnsNames":["raycluster-head-svc","localhost","raycluster-head-svc.test-ns-qbjjb.svc","raycluster-head-svc.test-ns-qbjjb.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:09:03.745Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22","name":"raycluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-qbjjb","name":"raycluster"}}
{"level":"info","ts":"2026-07-13T16:09:03.745Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22"}
{"level":"error","ts":"2026-07-13T16:09:03.745Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"b422f3db-21f1-4ef9-b48f-6211515dcb22","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:03.816Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2"}
{"level":"info","ts":"2026-07-13T16:09:03.816Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2"}
{"level":"info","ts":"2026-07-13T16:09:03.816Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:03.817Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2"}
{"level":"info","ts":"2026-07-13T16:09:03.817Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:09:03.817Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2"}
{"level":"info","ts":"2026-07-13T16:09:03.817Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:03.817Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2","cluster":"raycluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:09:03.817Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2","cluster":"raycluster","authType":"IntegratedOAuth","serviceAccount":"raycluster-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T16:09:03.817Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster","ray.io/group":"headgroup","ray.io/identifier":"raycluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:09:03.817Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T16:09:03.817Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T16:09:03.817Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T16:09:03.817Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2","enabled":true}
{"level":"error","ts":"2026-07-13T16:09:03.820Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:03.820Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:09:03.820Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:03.820Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:03.820Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:03.820Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated"}]}
{"level":"info","ts":"2026-07-13T16:09:03.820Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2","name":"raycluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:03Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:03Z","stateTransitionTimes":{"ready":"2026-07-13T15:49:35Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T15:49:35Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T15:49:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T15:49:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:09:03Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:09:03.832Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2"}
{"level":"error","ts":"2026-07-13T16:09:03.832Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"c916b1d2-6e96-4548-ab8e-33657ea79ea2","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:03.832Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"f88c7df0-ed2d-45cf-87de-e7121bf7c0ee","certificate":"ray-worker-cert-raycluster","dnsNames":["raycluster-worker-svc","localhost","raycluster-small-group","raycluster-worker-svc.test-ns-qbjjb.svc","raycluster-worker-svc.test-ns-qbjjb.svc.cluster.local","*.raycluster-worker-svc.test-ns-qbjjb.svc","*.raycluster-worker-svc.test-ns-qbjjb.svc.cluster.local","*-worker-*.test-ns-qbjjb.svc","*-worker-*.test-ns-qbjjb.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:09:03.832Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"f88c7df0-ed2d-45cf-87de-e7121bf7c0ee","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-07-13T16:09:03.833Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"9ac3fd69-5d38-453a-99fc-900148882a68","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:03.833Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"2dad22c7-a50e-4e4a-a72e-a495f617fc93","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:03.833Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"9ac3fd69-5d38-453a-99fc-900148882a68","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-07-13T16:09:03.836Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"2dad22c7-a50e-4e4a-a72e-a495f617fc93","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:03.932Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a807deb-5c36-435e-b991-0ea897133cb0"}
{"level":"info","ts":"2026-07-13T16:09:03.932Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a807deb-5c36-435e-b991-0ea897133cb0"}
{"level":"info","ts":"2026-07-13T16:09:03.932Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a807deb-5c36-435e-b991-0ea897133cb0","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:03.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a807deb-5c36-435e-b991-0ea897133cb0","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:03.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a807deb-5c36-435e-b991-0ea897133cb0","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:03.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a807deb-5c36-435e-b991-0ea897133cb0","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:09:03.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a807deb-5c36-435e-b991-0ea897133cb0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:03.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a807deb-5c36-435e-b991-0ea897133cb0","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:03.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a807deb-5c36-435e-b991-0ea897133cb0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:03.932Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a807deb-5c36-435e-b991-0ea897133cb0","podName":"ray-cluster-head-xmqjl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:03.932Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a807deb-5c36-435e-b991-0ea897133cb0","podName":"ray-cluster-head-xmqjl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:03.932Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a807deb-5c36-435e-b991-0ea897133cb0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:03.932Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a807deb-5c36-435e-b991-0ea897133cb0","seconds":300}
{"level":"info","ts":"2026-07-13T16:09:04.153Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2a30dc6d-4824-4e03-8736-a6be7a78cb18"}
{"level":"info","ts":"2026-07-13T16:09:04.153Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2a30dc6d-4824-4e03-8736-a6be7a78cb18"}
{"level":"info","ts":"2026-07-13T16:09:04.153Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2a30dc6d-4824-4e03-8736-a6be7a78cb18","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:04.153Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2a30dc6d-4824-4e03-8736-a6be7a78cb18"}
{"level":"info","ts":"2026-07-13T16:09:04.153Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2a30dc6d-4824-4e03-8736-a6be7a78cb18","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:09:04.153Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2a30dc6d-4824-4e03-8736-a6be7a78cb18"}
{"level":"info","ts":"2026-07-13T16:09:04.153Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2a30dc6d-4824-4e03-8736-a6be7a78cb18","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:04.153Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2a30dc6d-4824-4e03-8736-a6be7a78cb18","cluster":"raycluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:09:04.153Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2a30dc6d-4824-4e03-8736-a6be7a78cb18","cluster":"raycluster","authType":"IntegratedOAuth","serviceAccount":"raycluster-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T16:09:04.153Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2a30dc6d-4824-4e03-8736-a6be7a78cb18","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster","ray.io/group":"headgroup","ray.io/identifier":"raycluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:09:04.153Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2a30dc6d-4824-4e03-8736-a6be7a78cb18","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T16:09:04.153Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2a30dc6d-4824-4e03-8736-a6be7a78cb18","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T16:09:04.153Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2a30dc6d-4824-4e03-8736-a6be7a78cb18","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T16:09:04.153Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2a30dc6d-4824-4e03-8736-a6be7a78cb18","enabled":true}
{"level":"error","ts":"2026-07-13T16:09:04.157Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2a30dc6d-4824-4e03-8736-a6be7a78cb18","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:04.157Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2a30dc6d-4824-4e03-8736-a6be7a78cb18","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:09:04.157Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2a30dc6d-4824-4e03-8736-a6be7a78cb18","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:04.157Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2a30dc6d-4824-4e03-8736-a6be7a78cb18","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:04.157Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2a30dc6d-4824-4e03-8736-a6be7a78cb18","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:04.157Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2a30dc6d-4824-4e03-8736-a6be7a78cb18"}
{"level":"error","ts":"2026-07-13T16:09:04.157Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"2a30dc6d-4824-4e03-8736-a6be7a78cb18","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:04.798Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff90dfb7-035d-415c-b147-c8980c780e12"}
{"level":"info","ts":"2026-07-13T16:09:04.798Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff90dfb7-035d-415c-b147-c8980c780e12"}
{"level":"info","ts":"2026-07-13T16:09:04.798Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff90dfb7-035d-415c-b147-c8980c780e12","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:04.798Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff90dfb7-035d-415c-b147-c8980c780e12"}
{"level":"info","ts":"2026-07-13T16:09:04.798Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff90dfb7-035d-415c-b147-c8980c780e12","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:09:04.798Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff90dfb7-035d-415c-b147-c8980c780e12"}
{"level":"info","ts":"2026-07-13T16:09:04.798Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff90dfb7-035d-415c-b147-c8980c780e12","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:04.798Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff90dfb7-035d-415c-b147-c8980c780e12","cluster":"raycluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:09:04.798Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff90dfb7-035d-415c-b147-c8980c780e12","cluster":"raycluster","authType":"IntegratedOAuth","serviceAccount":"raycluster-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T16:09:04.798Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff90dfb7-035d-415c-b147-c8980c780e12","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster","ray.io/group":"headgroup","ray.io/identifier":"raycluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:09:04.798Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff90dfb7-035d-415c-b147-c8980c780e12","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T16:09:04.798Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff90dfb7-035d-415c-b147-c8980c780e12","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T16:09:04.798Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff90dfb7-035d-415c-b147-c8980c780e12","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T16:09:04.798Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff90dfb7-035d-415c-b147-c8980c780e12","enabled":true}
{"level":"error","ts":"2026-07-13T16:09:04.802Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff90dfb7-035d-415c-b147-c8980c780e12","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:04.802Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff90dfb7-035d-415c-b147-c8980c780e12","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:09:04.802Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff90dfb7-035d-415c-b147-c8980c780e12","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:04.802Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff90dfb7-035d-415c-b147-c8980c780e12","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:04.802Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff90dfb7-035d-415c-b147-c8980c780e12","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:04.802Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff90dfb7-035d-415c-b147-c8980c780e12"}
{"level":"error","ts":"2026-07-13T16:09:04.802Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"ff90dfb7-035d-415c-b147-c8980c780e12","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:05.580Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-x5qvg"},"reconcileID":"491bd96e-6873-4019-be52-ad18751bd0e2"}
{"level":"info","ts":"2026-07-13T16:09:06.083Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"710f6f33-1499-448f-8846-647e0bc31988"}
{"level":"info","ts":"2026-07-13T16:09:06.083Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"710f6f33-1499-448f-8846-647e0bc31988"}
{"level":"info","ts":"2026-07-13T16:09:06.083Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"710f6f33-1499-448f-8846-647e0bc31988","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:06.083Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"710f6f33-1499-448f-8846-647e0bc31988"}
{"level":"info","ts":"2026-07-13T16:09:06.083Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"710f6f33-1499-448f-8846-647e0bc31988","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:09:06.083Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"710f6f33-1499-448f-8846-647e0bc31988"}
{"level":"info","ts":"2026-07-13T16:09:06.083Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"710f6f33-1499-448f-8846-647e0bc31988","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:06.083Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"710f6f33-1499-448f-8846-647e0bc31988","cluster":"raycluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:09:06.083Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"710f6f33-1499-448f-8846-647e0bc31988","cluster":"raycluster","authType":"IntegratedOAuth","serviceAccount":"raycluster-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-07-13T16:09:06.083Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"710f6f33-1499-448f-8846-647e0bc31988","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster","ray.io/group":"headgroup","ray.io/identifier":"raycluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:09:06.083Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"710f6f33-1499-448f-8846-647e0bc31988","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-head-svc.test-ns-qbjjb.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-07-13T16:09:06.083Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"710f6f33-1499-448f-8846-647e0bc31988","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T16:09:06.083Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"710f6f33-1499-448f-8846-647e0bc31988","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-qbjjb.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-07-13T16:09:06.083Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"710f6f33-1499-448f-8846-647e0bc31988","enabled":true}
{"level":"error","ts":"2026-07-13T16:09:06.087Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"710f6f33-1499-448f-8846-647e0bc31988","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:06.087Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"710f6f33-1499-448f-8846-647e0bc31988","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:09:06.087Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"710f6f33-1499-448f-8846-647e0bc31988","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:06.087Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"710f6f33-1499-448f-8846-647e0bc31988","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:06.087Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"710f6f33-1499-448f-8846-647e0bc31988","filter labels":["test-ns-qbjjb",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:09:06.087Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"710f6f33-1499-448f-8846-647e0bc31988"}
{"level":"error","ts":"2026-07-13T16:09:06.087Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"710f6f33-1499-448f-8846-647e0bc31988","error":"FailedCreateHeadPod\npods \"raycluster-head-\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:07.782Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"7fea24a3-b72f-4c75-abc7-345af58e08d5","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:09:07.782Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"7fea24a3-b72f-4c75-abc7-345af58e08d5","ShutdownAfterJobFinishes":false,"ClusterSelector":{"ray.io/cluster":"raycluster"},"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 16:08:42 +0000 UTC","Now":"2026-07-13T16:09:07.782Z","ShutdownTime":"2026-07-13T16:08:42.000Z"}
{"level":"info","ts":"2026-07-13T16:09:07.784Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"b3c4ad16-1a6c-4c2d-a0e1-055ee91b970e","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-07-13T16:09:07.784Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"b3c4ad16-1a6c-4c2d-a0e1-055ee91b970e","ShutdownAfterJobFinishes":false,"ClusterSelector":{"ray.io/cluster":"raycluster"},"ttlSecondsAfterFinished":0,"Status.endTime":"2026-07-13 16:08:58 +0000 UTC","Now":"2026-07-13T16:09:07.784Z","ShutdownTime":"2026-07-13T16:08:58.000Z"}
{"level":"info","ts":"2026-07-13T16:09:07.940Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"05925df6-4b86-44ee-8fb7-c9e6641c0b23","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:07.944Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"05925df6-4b86-44ee-8fb7-c9e6641c0b23","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:07.944Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"14cddf60-35fc-4d2e-8f6f-8f543230e632","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:07.948Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"14cddf60-35fc-4d2e-8f6f-8f543230e632","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.225Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"2a43febf-e33b-4977-b5c0-6967de3883ea","namespacedName":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:09:08.225Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"2a43febf-e33b-4977-b5c0-6967de3883ea","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.225Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"2a43febf-e33b-4977-b5c0-6967de3883ea","cluster":"raycluster"}
{"level":"error","ts":"2026-07-13T16:09:08.241Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"2a43febf-e33b-4977-b5c0-6967de3883ea","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:08.241Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"2a43febf-e33b-4977-b5c0-6967de3883ea"}
{"level":"error","ts":"2026-07-13T16:09:08.241Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"2a43febf-e33b-4977-b5c0-6967de3883ea","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:08.241Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"f9c95468-5930-4b49-9b49-f4b321465145","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.241Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"1736a360-80c9-4e89-9374-9fcac1c426c9","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.241Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"f9c95468-5930-4b49-9b49-f4b321465145","rayCluster":"raycluster"}
{"level":"error","ts":"2026-07-13T16:09:08.243Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"f9c95468-5930-4b49-9b49-f4b321465145","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:09:08.243Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"f9c95468-5930-4b49-9b49-f4b321465145","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:08.243Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"f9c95468-5930-4b49-9b49-f4b321465145"}
{"level":"error","ts":"2026-07-13T16:09:08.243Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"f9c95468-5930-4b49-9b49-f4b321465145","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:08.245Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"1736a360-80c9-4e89-9374-9fcac1c426c9","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.246Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8a1b238b-178b-4646-8f4b-f508aa8239aa","namespacedName":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:09:08.247Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8a1b238b-178b-4646-8f4b-f508aa8239aa","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.247Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8a1b238b-178b-4646-8f4b-f508aa8239aa","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.248Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"31ef46b7-c3ae-479f-99c4-c09aaf12f865","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.248Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"31ef46b7-c3ae-479f-99c4-c09aaf12f865","rayCluster":"raycluster"}
{"level":"error","ts":"2026-07-13T16:09:08.250Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"31ef46b7-c3ae-479f-99c4-c09aaf12f865","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:09:08.250Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"31ef46b7-c3ae-479f-99c4-c09aaf12f865","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:08.250Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"31ef46b7-c3ae-479f-99c4-c09aaf12f865"}
{"level":"error","ts":"2026-07-13T16:09:08.250Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"31ef46b7-c3ae-479f-99c4-c09aaf12f865","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:09:08.256Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8a1b238b-178b-4646-8f4b-f508aa8239aa","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:08.256Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8a1b238b-178b-4646-8f4b-f508aa8239aa"}
{"level":"error","ts":"2026-07-13T16:09:08.256Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8a1b238b-178b-4646-8f4b-f508aa8239aa","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:08.261Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"81881c8a-5cf5-4409-ae31-d6186d317aa2","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.261Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"81881c8a-5cf5-4409-ae31-d6186d317aa2","rayCluster":"raycluster"}
{"level":"error","ts":"2026-07-13T16:09:08.263Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"81881c8a-5cf5-4409-ae31-d6186d317aa2","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:09:08.263Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"81881c8a-5cf5-4409-ae31-d6186d317aa2","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:08.263Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"81881c8a-5cf5-4409-ae31-d6186d317aa2"}
{"level":"error","ts":"2026-07-13T16:09:08.263Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"81881c8a-5cf5-4409-ae31-d6186d317aa2","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:08.266Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"640404b0-abe8-4976-bf6b-500d6f1b107f","namespacedName":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:09:08.266Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"640404b0-abe8-4976-bf6b-500d6f1b107f","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.266Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"640404b0-abe8-4976-bf6b-500d6f1b107f","cluster":"raycluster"}
{"level":"error","ts":"2026-07-13T16:09:08.276Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"640404b0-abe8-4976-bf6b-500d6f1b107f","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:08.276Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"640404b0-abe8-4976-bf6b-500d6f1b107f"}
{"level":"error","ts":"2026-07-13T16:09:08.276Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"640404b0-abe8-4976-bf6b-500d6f1b107f","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:08.284Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"455b5e65-91e2-421c-abe8-6f33ac28f921","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.284Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"455b5e65-91e2-421c-abe8-6f33ac28f921","rayCluster":"raycluster"}
{"level":"error","ts":"2026-07-13T16:09:08.287Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"455b5e65-91e2-421c-abe8-6f33ac28f921","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:09:08.287Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"455b5e65-91e2-421c-abe8-6f33ac28f921","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:08.287Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"455b5e65-91e2-421c-abe8-6f33ac28f921"}
{"level":"error","ts":"2026-07-13T16:09:08.287Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"455b5e65-91e2-421c-abe8-6f33ac28f921","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:08.297Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8762ea7c-a255-4fe3-aadf-b31bbd398592","namespacedName":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:09:08.297Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8762ea7c-a255-4fe3-aadf-b31bbd398592","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.297Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8762ea7c-a255-4fe3-aadf-b31bbd398592","cluster":"raycluster"}
{"level":"error","ts":"2026-07-13T16:09:08.307Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8762ea7c-a255-4fe3-aadf-b31bbd398592","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:08.307Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8762ea7c-a255-4fe3-aadf-b31bbd398592"}
{"level":"error","ts":"2026-07-13T16:09:08.307Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"8762ea7c-a255-4fe3-aadf-b31bbd398592","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:08.320Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"e6d68018-1b81-4cd1-bdbc-ca7240a486b9","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.321Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"e6d68018-1b81-4cd1-bdbc-ca7240a486b9","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.321Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"77f05bd6-bce2-4c74-aa8c-59a01146582b"}
{"level":"info","ts":"2026-07-13T16:09:08.321Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"5aab101a-625d-4064-a390-8831d290f19f","namespacedName":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:09:08.321Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"5aab101a-625d-4064-a390-8831d290f19f","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.321Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"5aab101a-625d-4064-a390-8831d290f19f","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.321Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"f2356528-0ad1-44de-be9b-022f233f1d17"}
{"level":"info","ts":"2026-07-13T16:09:08.321Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"5aab101a-625d-4064-a390-8831d290f19f","namespace":"test-ns-qbjjb","otherAuthClusters":0,"action":"will delete"}
{"level":"error","ts":"2026-07-13T16:09:08.323Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"e6d68018-1b81-4cd1-bdbc-ca7240a486b9","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:09:08.323Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"e6d68018-1b81-4cd1-bdbc-ca7240a486b9","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:08.323Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"e6d68018-1b81-4cd1-bdbc-ca7240a486b9"}
{"level":"error","ts":"2026-07-13T16:09:08.323Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"e6d68018-1b81-4cd1-bdbc-ca7240a486b9","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:08.325Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"5aab101a-625d-4064-a390-8831d290f19f","name":"kuberay-gateway-access","namespace":"test-ns-qbjjb"}
{"level":"info","ts":"2026-07-13T16:09:08.327Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a5a71048-180f-409d-bf4c-bc05c920f5c7","rayCluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.327Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a5a71048-180f-409d-bf4c-bc05c920f5c7","rayCluster":"raycluster"}
{"level":"error","ts":"2026-07-13T16:09:08.330Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a5a71048-180f-409d-bf4c-bc05c920f5c7","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:09:08.330Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a5a71048-180f-409d-bf4c-bc05c920f5c7","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:08.330Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a5a71048-180f-409d-bf4c-bc05c920f5c7"}
{"level":"error","ts":"2026-07-13T16:09:08.330Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a5a71048-180f-409d-bf4c-bc05c920f5c7","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster\" is forbidden: unable to create new content in namespace test-ns-qbjjb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:08.330Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"5aab101a-625d-4064-a390-8831d290f19f","name":"test-ns-qbjjb-raycluster","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-13T16:09:08.336Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"reconcileID":"95c6dc48-2177-41f7-b09d-3f245322ff1a"}
{"level":"info","ts":"2026-07-13T16:09:08.337Z","logger":"raycluster-resource","msg":"default","name":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.337Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster","namespace":"test-ns-qbjjb"}
{"level":"info","ts":"2026-07-13T16:09:08.350Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"5aab101a-625d-4064-a390-8831d290f19f","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.350Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a0438ed0-e0ff-4914-b6a3-e65fc5537217","namespacedName":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:09:08.350Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a0438ed0-e0ff-4914-b6a3-e65fc5537217","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.350Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a0438ed0-e0ff-4914-b6a3-e65fc5537217","cluster":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.350Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a0438ed0-e0ff-4914-b6a3-e65fc5537217","namespace":"test-ns-qbjjb","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-13T16:09:08.350Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a0438ed0-e0ff-4914-b6a3-e65fc5537217","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-13T16:09:08.350Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"604830c0-fbe3-464f-be96-ba3b56d75580","namespace":"test-ns-qbjjb","clusterName":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.350Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"604830c0-fbe3-464f-be96-ba3b56d75580","namespace":"test-ns-qbjjb","clusterName":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.350Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"3ae55bfb-9331-401f-a335-4f476e161cf5"}
{"level":"error","ts":"2026-07-13T16:09:08.354Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a0438ed0-e0ff-4914-b6a3-e65fc5537217","error":"rayclusters.ray.io \"raycluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:08.354Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a0438ed0-e0ff-4914-b6a3-e65fc5537217"}
{"level":"error","ts":"2026-07-13T16:09:08.354Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"a0438ed0-e0ff-4914-b6a3-e65fc5537217","error":"rayclusters.ray.io \"raycluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:08.354Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"0a863291-5006-4c49-a86f-47574ebb59d7","namespacedName":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:09:08.354Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"0a863291-5006-4c49-a86f-47574ebb59d7"}
{"level":"info","ts":"2026-07-13T16:09:08.354Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"0a863291-5006-4c49-a86f-47574ebb59d7","namespace":"test-ns-qbjjb","authClusters":0}
{"level":"info","ts":"2026-07-13T16:09:08.354Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"0a863291-5006-4c49-a86f-47574ebb59d7","namespace":"test-ns-qbjjb"}
{"level":"info","ts":"2026-07-13T16:09:08.359Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"2dfcbcfd-04cf-4af2-ab1b-810b253ea5ff","namespacedName":{"name":"raycluster","namespace":"test-ns-qbjjb"}}
{"level":"info","ts":"2026-07-13T16:09:08.359Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"2dfcbcfd-04cf-4af2-ab1b-810b253ea5ff"}
{"level":"info","ts":"2026-07-13T16:09:08.359Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"2dfcbcfd-04cf-4af2-ab1b-810b253ea5ff","namespace":"test-ns-qbjjb","authClusters":0}
{"level":"info","ts":"2026-07-13T16:09:08.359Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"2dfcbcfd-04cf-4af2-ab1b-810b253ea5ff","namespace":"test-ns-qbjjb"}
{"level":"info","ts":"2026-07-13T16:09:08.362Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"287a6182-8c26-4e26-aeb1-36bbcc8f7ee2"}
{"level":"info","ts":"2026-07-13T16:09:08.362Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"287a6182-8c26-4e26-aeb1-36bbcc8f7ee2"}
{"level":"info","ts":"2026-07-13T16:09:08.362Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"287a6182-8c26-4e26-aeb1-36bbcc8f7ee2","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:08.362Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"287a6182-8c26-4e26-aeb1-36bbcc8f7ee2","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:08.362Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"287a6182-8c26-4e26-aeb1-36bbcc8f7ee2","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:08.362Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"287a6182-8c26-4e26-aeb1-36bbcc8f7ee2","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:09:08.362Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"287a6182-8c26-4e26-aeb1-36bbcc8f7ee2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:08.362Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"287a6182-8c26-4e26-aeb1-36bbcc8f7ee2","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:08.362Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"287a6182-8c26-4e26-aeb1-36bbcc8f7ee2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:08.362Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"287a6182-8c26-4e26-aeb1-36bbcc8f7ee2","podName":"ray-cluster-head-xmqjl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:08.362Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"287a6182-8c26-4e26-aeb1-36bbcc8f7ee2","podName":"ray-cluster-head-xmqjl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:08.362Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"287a6182-8c26-4e26-aeb1-36bbcc8f7ee2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:08.362Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"287a6182-8c26-4e26-aeb1-36bbcc8f7ee2","seconds":300}
{"level":"info","ts":"2026-07-13T16:09:08.491Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"055c864c-6ea5-4327-a3e5-373845849be6","namespace":"test-ns-qbjjb","clusterName":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.491Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-qbjjb"},"namespace":"test-ns-qbjjb","name":"raycluster","reconcileID":"055c864c-6ea5-4327-a3e5-373845849be6","namespace":"test-ns-qbjjb","clusterName":"raycluster"}
{"level":"info","ts":"2026-07-13T16:09:08.624Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"5d54fb55-75ee-48c4-8afa-402b2b85993e","DeletionTimestamp":"2026-07-13 16:09:08 +0000 UTC"}
{"level":"info","ts":"2026-07-13T16:09:08.624Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"5d54fb55-75ee-48c4-8afa-402b2b85993e","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T16:09:08.635Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"b8c3ebe4-80c0-4223-aa8c-ba35b3cab341","DeletionTimestamp":"2026-07-13 16:09:08 +0000 UTC"}
{"level":"info","ts":"2026-07-13T16:09:08.635Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"b8c3ebe4-80c0-4223-aa8c-ba35b3cab341","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-07-13T16:09:08.644Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"cdf5f063-dda1-4958-aed4-c7a0f34fe248"}
{"level":"info","ts":"2026-07-13T16:09:08.645Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"managed-externally","namespace":"test-ns-qbjjb"},"reconcileID":"f65551fa-07ac-4c8c-812f-454666acb61c"}
{"level":"info","ts":"2026-07-13T16:09:08.645Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"90667680-e04e-4cfa-906d-e627525d70e5"}
{"level":"info","ts":"2026-07-13T16:09:09.369Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4326ea8e-075e-466b-a422-eaca4c7625da"}
{"level":"info","ts":"2026-07-13T16:09:09.369Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4326ea8e-075e-466b-a422-eaca4c7625da"}
{"level":"info","ts":"2026-07-13T16:09:09.370Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4326ea8e-075e-466b-a422-eaca4c7625da","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:09.370Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4326ea8e-075e-466b-a422-eaca4c7625da","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:09.370Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4326ea8e-075e-466b-a422-eaca4c7625da","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:09.370Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4326ea8e-075e-466b-a422-eaca4c7625da","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:09:09.370Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4326ea8e-075e-466b-a422-eaca4c7625da","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:09.370Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4326ea8e-075e-466b-a422-eaca4c7625da","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:09.370Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4326ea8e-075e-466b-a422-eaca4c7625da","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:09.370Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4326ea8e-075e-466b-a422-eaca4c7625da","podName":"ray-cluster-head-xmqjl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:09:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:09:09.370Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4326ea8e-075e-466b-a422-eaca4c7625da","podName":"ray-cluster-head-xmqjl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:09:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:09:09.370Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4326ea8e-075e-466b-a422-eaca4c7625da","old conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"new conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}]}
{"level":"info","ts":"2026-07-13T16:09:09.370Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4326ea8e-075e-466b-a422-eaca4c7625da","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:09:09.381Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d709c863-d011-4463-9c86-df864d108d09","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:09.381Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"b03fc661-ad61-4675-b283-b9906fc4c093","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:09.381Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"b03fc661-ad61-4675-b283-b9906fc4c093","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:09:09.382Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"b03fc661-ad61-4675-b283-b9906fc4c093","rayCluster":"ray-cluster","podIPs":["10.133.0.40"]}
{"level":"info","ts":"2026-07-13T16:09:09.397Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d709c863-d011-4463-9c86-df864d108d09","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:09:09.409Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d709c863-d011-4463-9c86-df864d108d09","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:09:09.409Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d709c863-d011-4463-9c86-df864d108d09","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:11.381Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"760cef0c-150c-46ca-84e7-05200e56e038"}
{"level":"info","ts":"2026-07-13T16:09:11.381Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"760cef0c-150c-46ca-84e7-05200e56e038"}
{"level":"info","ts":"2026-07-13T16:09:11.381Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"760cef0c-150c-46ca-84e7-05200e56e038","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:11.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"760cef0c-150c-46ca-84e7-05200e56e038","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:11.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"760cef0c-150c-46ca-84e7-05200e56e038","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:11.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"760cef0c-150c-46ca-84e7-05200e56e038","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:09:11.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"760cef0c-150c-46ca-84e7-05200e56e038","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:11.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"760cef0c-150c-46ca-84e7-05200e56e038","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:11.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"760cef0c-150c-46ca-84e7-05200e56e038","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:11.381Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"760cef0c-150c-46ca-84e7-05200e56e038","podName":"ray-cluster-head-xmqjl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:09:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:09:11.382Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"760cef0c-150c-46ca-84e7-05200e56e038","podName":"ray-cluster-head-xmqjl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:09:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:09:11.382Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"760cef0c-150c-46ca-84e7-05200e56e038","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:11.382Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"760cef0c-150c-46ca-84e7-05200e56e038","seconds":300}
{"level":"info","ts":"2026-07-13T16:09:11.635Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-qbjjb"},"reconcileID":"ec6d9085-8ebc-4f5a-b1cf-7c9966acba85"}
{"level":"info","ts":"2026-07-13T16:09:11.645Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-qbjjb"},"reconcileID":"27d5a092-e428-4208-a7d8-3ae37a095434"}
{"level":"info","ts":"2026-07-13T16:09:25.398Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8b95e439-85d9-4968-91a9-9f191ba05ecd"}
{"level":"info","ts":"2026-07-13T16:09:25.398Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8b95e439-85d9-4968-91a9-9f191ba05ecd"}
{"level":"info","ts":"2026-07-13T16:09:25.398Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8b95e439-85d9-4968-91a9-9f191ba05ecd","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:25.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8b95e439-85d9-4968-91a9-9f191ba05ecd","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:25.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8b95e439-85d9-4968-91a9-9f191ba05ecd","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:25.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8b95e439-85d9-4968-91a9-9f191ba05ecd","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:09:25.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8b95e439-85d9-4968-91a9-9f191ba05ecd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:25.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8b95e439-85d9-4968-91a9-9f191ba05ecd","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:25.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8b95e439-85d9-4968-91a9-9f191ba05ecd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:25.398Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8b95e439-85d9-4968-91a9-9f191ba05ecd","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-13T16:09:25.398Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8b95e439-85d9-4968-91a9-9f191ba05ecd","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:25Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:09:25.410Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"6efe7a17-c74b-4bb4-9aa5-2ef9f89c4355","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:25.410Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"e1b420d4-f8bf-4b1f-9fe9-29511037075b","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:25.410Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"6efe7a17-c74b-4bb4-9aa5-2ef9f89c4355","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:09:25.410Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"6efe7a17-c74b-4bb4-9aa5-2ef9f89c4355","rayCluster":"ray-cluster","podIPs":["10.133.0.40"]}
{"level":"info","ts":"2026-07-13T16:09:25.419Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"e1b420d4-f8bf-4b1f-9fe9-29511037075b","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:09:25.429Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"e1b420d4-f8bf-4b1f-9fe9-29511037075b","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:09:25.429Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"e1b420d4-f8bf-4b1f-9fe9-29511037075b","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:27.410Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3ef4de8b-b792-4637-9ab1-4ca6d626b95c"}
{"level":"info","ts":"2026-07-13T16:09:27.410Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3ef4de8b-b792-4637-9ab1-4ca6d626b95c"}
{"level":"info","ts":"2026-07-13T16:09:27.410Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3ef4de8b-b792-4637-9ab1-4ca6d626b95c","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:27.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3ef4de8b-b792-4637-9ab1-4ca6d626b95c","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:27.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3ef4de8b-b792-4637-9ab1-4ca6d626b95c","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:27.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3ef4de8b-b792-4637-9ab1-4ca6d626b95c","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:09:27.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3ef4de8b-b792-4637-9ab1-4ca6d626b95c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:27.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3ef4de8b-b792-4637-9ab1-4ca6d626b95c","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:27.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3ef4de8b-b792-4637-9ab1-4ca6d626b95c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:27.410Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3ef4de8b-b792-4637-9ab1-4ca6d626b95c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:27.410Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3ef4de8b-b792-4637-9ab1-4ca6d626b95c","seconds":300}
{"level":"info","ts":"2026-07-13T16:09:30.026Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-x5qvg"},"reconcileID":"dccd5569-60d3-4e36-a13a-9c279cbdb6ce"}
{"level":"info","ts":"2026-07-13T16:09:30.427Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"137b11dc-fd16-4156-8e26-5575a35ae8db","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:30.427Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"137b11dc-fd16-4156-8e26-5575a35ae8db","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:09:30.427Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"137b11dc-fd16-4156-8e26-5575a35ae8db","rayCluster":"ray-cluster","podIPs":["10.133.0.40"]}
{"level":"info","ts":"2026-07-13T16:09:31.182Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:31.183Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-9wtq2"}
{"level":"info","ts":"2026-07-13T16:09:31.193Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"edc16d58-efb7-4e5c-8bac-b4700dd1cadc","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:31.193Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d4274543-6373-405b-bc31-846a946d7806","namespacedName":{"name":"ray-cluster","namespace":"test-ns-9wtq2"}}
{"level":"info","ts":"2026-07-13T16:09:31.193Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"9dfc02dd-10f7-4216-b841-9a712ddc21d1","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:31.193Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d4274543-6373-405b-bc31-846a946d7806","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:31.193Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d4274543-6373-405b-bc31-846a946d7806","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:31.193Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"edc16d58-efb7-4e5c-8bac-b4700dd1cadc","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:09:31.193Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1dc254d9-570b-472d-96d3-2ea8b55e0261"}
{"level":"info","ts":"2026-07-13T16:09:31.193Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1dc254d9-570b-472d-96d3-2ea8b55e0261"}
{"level":"info","ts":"2026-07-13T16:09:31.193Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1dc254d9-570b-472d-96d3-2ea8b55e0261","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:31.193Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"edc16d58-efb7-4e5c-8bac-b4700dd1cadc","rayCluster":"ray-cluster","podIPs":["10.133.0.40"]}
{"level":"info","ts":"2026-07-13T16:09:31.193Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1dc254d9-570b-472d-96d3-2ea8b55e0261","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:31.194Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1dc254d9-570b-472d-96d3-2ea8b55e0261","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:31.194Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1dc254d9-570b-472d-96d3-2ea8b55e0261","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-13T16:09:31.194Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1dc254d9-570b-472d-96d3-2ea8b55e0261","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:31.194Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1dc254d9-570b-472d-96d3-2ea8b55e0261","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:09:31.194Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1dc254d9-570b-472d-96d3-2ea8b55e0261","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:31.194Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1dc254d9-570b-472d-96d3-2ea8b55e0261","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:09:31.194Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1dc254d9-570b-472d-96d3-2ea8b55e0261"}
{"level":"info","ts":"2026-07-13T16:09:31.194Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1dc254d9-570b-472d-96d3-2ea8b55e0261","nodeType":"worker","rayStartParams":{"address":"ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"1G"},"requests":{"cpu":"1","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:09:31.194Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1dc254d9-570b-472d-96d3-2ea8b55e0261","rayStartCmd":"ray start  --address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:09:31.194Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1dc254d9-570b-472d-96d3-2ea8b55e0261","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:09:31.194Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1dc254d9-570b-472d-96d3-2ea8b55e0261","enabled":true}
{"level":"info","ts":"2026-07-13T16:09:31.206Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d4274543-6373-405b-bc31-846a946d7806","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:31.206Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"2967d1ce-a54c-4ce9-ab5e-f76500e77a0f","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:31.206Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"2967d1ce-a54c-4ce9-ab5e-f76500e77a0f","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:09:31.206Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"2967d1ce-a54c-4ce9-ab5e-f76500e77a0f","rayCluster":"ray-cluster","podIPs":["10.133.0.40"]}
{"level":"info","ts":"2026-07-13T16:09:31.208Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1dc254d9-570b-472d-96d3-2ea8b55e0261","name":"ray-cluster-small-group-worker-tqz5h"}
{"level":"info","ts":"2026-07-13T16:09:31.208Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1dc254d9-570b-472d-96d3-2ea8b55e0261","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:09:31.208Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1dc254d9-570b-472d-96d3-2ea8b55e0261","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:25Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:31Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-07-13T16:09:31.208Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"9dfc02dd-10f7-4216-b841-9a712ddc21d1","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:09:31.216Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1dc254d9-570b-472d-96d3-2ea8b55e0261","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-9wtq2","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-13T16:09:31.216Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1dc254d9-570b-472d-96d3-2ea8b55e0261"}
{"level":"error","ts":"2026-07-13T16:09:31.216Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1dc254d9-570b-472d-96d3-2ea8b55e0261","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:31.216Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5a64b36c-c0c6-4b65-b529-5b1a1bcdeba2"}
{"level":"info","ts":"2026-07-13T16:09:31.216Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5a64b36c-c0c6-4b65-b529-5b1a1bcdeba2"}
{"level":"info","ts":"2026-07-13T16:09:31.216Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5a64b36c-c0c6-4b65-b529-5b1a1bcdeba2","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:31.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5a64b36c-c0c6-4b65-b529-5b1a1bcdeba2","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:31.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5a64b36c-c0c6-4b65-b529-5b1a1bcdeba2","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:31.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5a64b36c-c0c6-4b65-b529-5b1a1bcdeba2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-13T16:09:31.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5a64b36c-c0c6-4b65-b529-5b1a1bcdeba2","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:31.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5a64b36c-c0c6-4b65-b529-5b1a1bcdeba2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:31.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5a64b36c-c0c6-4b65-b529-5b1a1bcdeba2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:31.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5a64b36c-c0c6-4b65-b529-5b1a1bcdeba2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:31.221Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5a64b36c-c0c6-4b65-b529-5b1a1bcdeba2","podName":"ray-cluster-small-group-worker-tqz5h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:31.221Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5a64b36c-c0c6-4b65-b529-5b1a1bcdeba2","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:09:31.221Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5a64b36c-c0c6-4b65-b529-5b1a1bcdeba2","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:25Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:31Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-07-13T16:09:31.223Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"9dfc02dd-10f7-4216-b841-9a712ddc21d1","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:09:31.223Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"9dfc02dd-10f7-4216-b841-9a712ddc21d1","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:31.223Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"e7764fcb-abf5-4143-b4a1-e4c41d89ae09","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:31.233Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"aa5972eb-b638-4671-92d1-4aaa9540d6e9"}
{"level":"info","ts":"2026-07-13T16:09:31.233Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"aa5972eb-b638-4671-92d1-4aaa9540d6e9"}
{"level":"info","ts":"2026-07-13T16:09:31.233Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"aa5972eb-b638-4671-92d1-4aaa9540d6e9","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:31.233Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"aa5972eb-b638-4671-92d1-4aaa9540d6e9","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:31.233Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"aa5972eb-b638-4671-92d1-4aaa9540d6e9","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:31.233Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"aa5972eb-b638-4671-92d1-4aaa9540d6e9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-13T16:09:31.233Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"aa5972eb-b638-4671-92d1-4aaa9540d6e9","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:31.233Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"aa5972eb-b638-4671-92d1-4aaa9540d6e9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:31.233Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"aa5972eb-b638-4671-92d1-4aaa9540d6e9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:31.233Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"aa5972eb-b638-4671-92d1-4aaa9540d6e9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:31.234Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"aa5972eb-b638-4671-92d1-4aaa9540d6e9","podName":"ray-cluster-small-group-worker-tqz5h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:31.234Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"aa5972eb-b638-4671-92d1-4aaa9540d6e9","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:09:31.234Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"aa5972eb-b638-4671-92d1-4aaa9540d6e9","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:25Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:31Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-07-13T16:09:31.234Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d628d307-edbf-4c3f-84c5-e3fe935e5ddf","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:31.234Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d628d307-edbf-4c3f-84c5-e3fe935e5ddf","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:09:31.234Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d628d307-edbf-4c3f-84c5-e3fe935e5ddf","rayCluster":"ray-cluster","podIPs":["10.133.0.40"]}
{"level":"info","ts":"2026-07-13T16:09:31.239Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"e7764fcb-abf5-4143-b4a1-e4c41d89ae09","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:09:31.240Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"aa5972eb-b638-4671-92d1-4aaa9540d6e9","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-9wtq2","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-13T16:09:31.240Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"aa5972eb-b638-4671-92d1-4aaa9540d6e9"}
{"level":"error","ts":"2026-07-13T16:09:31.240Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"aa5972eb-b638-4671-92d1-4aaa9540d6e9","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:31.246Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f9527655-663b-4ee3-b55c-4b91ce06141f"}
{"level":"info","ts":"2026-07-13T16:09:31.246Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f9527655-663b-4ee3-b55c-4b91ce06141f"}
{"level":"info","ts":"2026-07-13T16:09:31.246Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f9527655-663b-4ee3-b55c-4b91ce06141f","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:31.246Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f9527655-663b-4ee3-b55c-4b91ce06141f","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:31.246Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f9527655-663b-4ee3-b55c-4b91ce06141f","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:31.246Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f9527655-663b-4ee3-b55c-4b91ce06141f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-13T16:09:31.246Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f9527655-663b-4ee3-b55c-4b91ce06141f","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:31.246Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f9527655-663b-4ee3-b55c-4b91ce06141f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:31.246Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f9527655-663b-4ee3-b55c-4b91ce06141f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:31.246Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f9527655-663b-4ee3-b55c-4b91ce06141f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:31.246Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f9527655-663b-4ee3-b55c-4b91ce06141f","podName":"ray-cluster-small-group-worker-tqz5h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:31.246Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f9527655-663b-4ee3-b55c-4b91ce06141f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:31.246Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"f9527655-663b-4ee3-b55c-4b91ce06141f","seconds":300}
{"level":"info","ts":"2026-07-13T16:09:31.248Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9b99038b-58a6-4aab-808d-dde88d71a04b"}
{"level":"info","ts":"2026-07-13T16:09:31.248Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9b99038b-58a6-4aab-808d-dde88d71a04b"}
{"level":"info","ts":"2026-07-13T16:09:31.248Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9b99038b-58a6-4aab-808d-dde88d71a04b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:31.248Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9b99038b-58a6-4aab-808d-dde88d71a04b","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:31.248Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9b99038b-58a6-4aab-808d-dde88d71a04b","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:31.248Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9b99038b-58a6-4aab-808d-dde88d71a04b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-13T16:09:31.248Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9b99038b-58a6-4aab-808d-dde88d71a04b","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:31.248Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9b99038b-58a6-4aab-808d-dde88d71a04b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:31.248Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9b99038b-58a6-4aab-808d-dde88d71a04b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:31.248Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9b99038b-58a6-4aab-808d-dde88d71a04b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:31.248Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9b99038b-58a6-4aab-808d-dde88d71a04b","podName":"ray-cluster-small-group-worker-tqz5h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:31.248Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9b99038b-58a6-4aab-808d-dde88d71a04b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:31.248Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9b99038b-58a6-4aab-808d-dde88d71a04b","seconds":300}
{"level":"info","ts":"2026-07-13T16:09:31.251Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"e7764fcb-abf5-4143-b4a1-e4c41d89ae09","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:09:31.251Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"e7764fcb-abf5-4143-b4a1-e4c41d89ae09","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:31.251Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"3f0e697e-0034-4b3c-aabc-1d70a6a71545","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:31.260Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"3f0e697e-0034-4b3c-aabc-1d70a6a71545","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:09:31.269Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"3f0e697e-0034-4b3c-aabc-1d70a6a71545","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:09:31.269Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"3f0e697e-0034-4b3c-aabc-1d70a6a71545","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:31.680Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"058b62ab-030b-487d-b082-8ef79aa70bc8"}
{"level":"info","ts":"2026-07-13T16:09:31.680Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"058b62ab-030b-487d-b082-8ef79aa70bc8"}
{"level":"info","ts":"2026-07-13T16:09:31.681Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"058b62ab-030b-487d-b082-8ef79aa70bc8","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:31.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"058b62ab-030b-487d-b082-8ef79aa70bc8","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:31.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"058b62ab-030b-487d-b082-8ef79aa70bc8","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:31.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"058b62ab-030b-487d-b082-8ef79aa70bc8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-13T16:09:31.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"058b62ab-030b-487d-b082-8ef79aa70bc8","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:31.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"058b62ab-030b-487d-b082-8ef79aa70bc8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:31.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"058b62ab-030b-487d-b082-8ef79aa70bc8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:31.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"058b62ab-030b-487d-b082-8ef79aa70bc8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:31.681Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"058b62ab-030b-487d-b082-8ef79aa70bc8","podName":"ray-cluster-small-group-worker-tqz5h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:31.681Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"058b62ab-030b-487d-b082-8ef79aa70bc8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:31.681Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"058b62ab-030b-487d-b082-8ef79aa70bc8","seconds":300}
{"level":"info","ts":"2026-07-13T16:09:31.821Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"369fc55a-f4c0-44b6-9dcd-c021a08b37aa"}
{"level":"info","ts":"2026-07-13T16:09:31.821Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"369fc55a-f4c0-44b6-9dcd-c021a08b37aa"}
{"level":"info","ts":"2026-07-13T16:09:31.821Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"369fc55a-f4c0-44b6-9dcd-c021a08b37aa","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:31.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"369fc55a-f4c0-44b6-9dcd-c021a08b37aa","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:31.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"369fc55a-f4c0-44b6-9dcd-c021a08b37aa","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:31.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"369fc55a-f4c0-44b6-9dcd-c021a08b37aa","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-13T16:09:31.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"369fc55a-f4c0-44b6-9dcd-c021a08b37aa","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:31.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"369fc55a-f4c0-44b6-9dcd-c021a08b37aa","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:31.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"369fc55a-f4c0-44b6-9dcd-c021a08b37aa","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:31.822Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"369fc55a-f4c0-44b6-9dcd-c021a08b37aa","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:31.822Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"369fc55a-f4c0-44b6-9dcd-c021a08b37aa","podName":"ray-cluster-small-group-worker-tqz5h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:31.822Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"369fc55a-f4c0-44b6-9dcd-c021a08b37aa","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:31.822Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"369fc55a-f4c0-44b6-9dcd-c021a08b37aa","seconds":300}
{"level":"info","ts":"2026-07-13T16:09:36.304Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:36.304Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-9wtq2"}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"1409ef64-c0db-4356-b5bd-6e523f84b50a","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"11527de6-04c9-4ef3-b1db-d2b116b75625","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"2305938f-4f13-4b18-8278-688aa500699f","namespacedName":{"name":"ray-cluster","namespace":"test-ns-9wtq2"}}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"1409ef64-c0db-4356-b5bd-6e523f84b50a","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"1409ef64-c0db-4356-b5bd-6e523f84b50a","pod":"ray-cluster-small-group-worker-tqz5h","ip":"10.134.0.37"}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"2305938f-4f13-4b18-8278-688aa500699f","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"2305938f-4f13-4b18-8278-688aa500699f","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896"}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896"}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896","workerReplicas":2,"NumOfHosts":1,"runningPods":1,"diff":1}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896"}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896","nodeType":"worker","rayStartParams":{"address":"ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"1G"},"requests":{"cpu":"1","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896","rayStartCmd":"ray start  --address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:09:36.315Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896","enabled":true}
{"level":"info","ts":"2026-07-13T16:09:36.325Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"11527de6-04c9-4ef3-b1db-d2b116b75625","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:09:36.326Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"1409ef64-c0db-4356-b5bd-6e523f84b50a","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-9wtq2.svc","ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local"],"ipAddresses":["10.133.0.40","10.134.0.37","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:09:36.330Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"2305938f-4f13-4b18-8278-688aa500699f","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:36.331Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896","name":"ray-cluster-small-group-worker-z7gbg"}
{"level":"info","ts":"2026-07-13T16:09:36.332Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896","podName":"ray-cluster-small-group-worker-tqz5h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:36.332Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:09:36.332Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:31Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:36Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-13T16:09:36.337Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"1409ef64-c0db-4356-b5bd-6e523f84b50a","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-9wtq2.svc","ray-cluster-worker-svc.test-ns-9wtq2.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-9wtq2.svc","*.ray-cluster-worker-svc.test-ns-9wtq2.svc.cluster.local","*-worker-*.test-ns-9wtq2.svc","*-worker-*.test-ns-9wtq2.svc.cluster.local"],"ipAddresses":["10.133.0.40","10.134.0.37","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:09:36.337Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"1409ef64-c0db-4356-b5bd-6e523f84b50a","rayCluster":"ray-cluster","podIPs":["10.133.0.40","10.134.0.37"]}
{"level":"info","ts":"2026-07-13T16:09:36.337Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0c22e1fe-d775-4990-8458-345e1307f613","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:36.337Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0c22e1fe-d775-4990-8458-345e1307f613","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:09:36.337Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0c22e1fe-d775-4990-8458-345e1307f613","pod":"ray-cluster-small-group-worker-tqz5h","ip":"10.134.0.37"}
{"level":"info","ts":"2026-07-13T16:09:36.339Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"11527de6-04c9-4ef3-b1db-d2b116b75625","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:09:36.339Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"11527de6-04c9-4ef3-b1db-d2b116b75625","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:36.339Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0e29e833-c266-4e75-880f-22b22cc1c480","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:36.340Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-9wtq2","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-13T16:09:36.340Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896"}
{"level":"error","ts":"2026-07-13T16:09:36.340Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"35daa7b6-a10f-4429-b9c7-0de8e33c8896","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:36.340Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0c04937f-6788-4b1a-afe4-339184dd3340"}
{"level":"info","ts":"2026-07-13T16:09:36.340Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0c04937f-6788-4b1a-afe4-339184dd3340"}
{"level":"info","ts":"2026-07-13T16:09:36.341Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0c04937f-6788-4b1a-afe4-339184dd3340","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:36.341Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0c04937f-6788-4b1a-afe4-339184dd3340","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:36.341Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0c04937f-6788-4b1a-afe4-339184dd3340","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:36.341Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0c04937f-6788-4b1a-afe4-339184dd3340","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:09:36.341Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0c04937f-6788-4b1a-afe4-339184dd3340","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:36.341Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0c04937f-6788-4b1a-afe4-339184dd3340","worker Pod":"ray-cluster-small-group-worker-z7gbg","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-z7gbg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:36.341Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0c04937f-6788-4b1a-afe4-339184dd3340","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:36.341Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0c04937f-6788-4b1a-afe4-339184dd3340","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:36.341Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0c04937f-6788-4b1a-afe4-339184dd3340","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:36.341Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0c04937f-6788-4b1a-afe4-339184dd3340","podName":"ray-cluster-small-group-worker-z7gbg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:36.341Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0c04937f-6788-4b1a-afe4-339184dd3340","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:09:36.341Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0c04937f-6788-4b1a-afe4-339184dd3340","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:31Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:36Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"error","ts":"2026-07-13T16:09:36.341Z","msg":"Failed to update certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0c22e1fe-d775-4990-8458-345e1307f613","error":"Operation cannot be fulfilled on certificates.cert-manager.io \"ray-worker-cert-ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).updateCertificateWithPodIPs\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:465\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:212\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:09:36.341Z","msg":"Failed to update worker certificate with pod IPs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0c22e1fe-d775-4990-8458-345e1307f613","error":"Operation cannot be fulfilled on certificates.cert-manager.io \"ray-worker-cert-ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:213\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:36.341Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0c22e1fe-d775-4990-8458-345e1307f613"}
{"level":"error","ts":"2026-07-13T16:09:36.341Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0c22e1fe-d775-4990-8458-345e1307f613","error":"Operation cannot be fulfilled on certificates.cert-manager.io \"ray-worker-cert-ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:36.347Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"1d926f73-cad7-4690-ae90-504be1afabd7","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:36.347Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"1d926f73-cad7-4690-ae90-504be1afabd7","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:09:36.347Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"1d926f73-cad7-4690-ae90-504be1afabd7","pod":"ray-cluster-small-group-worker-tqz5h","ip":"10.134.0.37"}
{"level":"info","ts":"2026-07-13T16:09:36.347Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"1d926f73-cad7-4690-ae90-504be1afabd7"}
{"level":"info","ts":"2026-07-13T16:09:36.354Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0e29e833-c266-4e75-880f-22b22cc1c480","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:09:36.356Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"650f0a91-efc1-46d0-8d81-89df055de44e"}
{"level":"info","ts":"2026-07-13T16:09:36.421Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"650f0a91-efc1-46d0-8d81-89df055de44e"}
{"level":"info","ts":"2026-07-13T16:09:36.421Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"4ab9ec6b-a384-471d-8ced-cd07575ca65d","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:36.421Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"650f0a91-efc1-46d0-8d81-89df055de44e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:36.421Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"4ab9ec6b-a384-471d-8ced-cd07575ca65d","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:09:36.421Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"4ab9ec6b-a384-471d-8ced-cd07575ca65d","pod":"ray-cluster-small-group-worker-tqz5h","ip":"10.134.0.37"}
{"level":"info","ts":"2026-07-13T16:09:36.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"650f0a91-efc1-46d0-8d81-89df055de44e","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:36.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"650f0a91-efc1-46d0-8d81-89df055de44e","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:36.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"650f0a91-efc1-46d0-8d81-89df055de44e","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:09:36.421Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"4ab9ec6b-a384-471d-8ced-cd07575ca65d"}
{"level":"info","ts":"2026-07-13T16:09:36.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"650f0a91-efc1-46d0-8d81-89df055de44e","worker Pod":"ray-cluster-small-group-worker-z7gbg","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-z7gbg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:36.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"650f0a91-efc1-46d0-8d81-89df055de44e","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:36.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"650f0a91-efc1-46d0-8d81-89df055de44e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:36.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"650f0a91-efc1-46d0-8d81-89df055de44e","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:36.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"650f0a91-efc1-46d0-8d81-89df055de44e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:36.421Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"650f0a91-efc1-46d0-8d81-89df055de44e","podName":"ray-cluster-small-group-worker-tqz5h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:36.421Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"650f0a91-efc1-46d0-8d81-89df055de44e","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:09:36.421Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"650f0a91-efc1-46d0-8d81-89df055de44e","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:31Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:36Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-13T16:09:36.425Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0e29e833-c266-4e75-880f-22b22cc1c480","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:09:36.425Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0e29e833-c266-4e75-880f-22b22cc1c480","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:36.425Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"fc6a4d73-7f7c-4951-b987-7da676a022cf","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:36.429Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"650f0a91-efc1-46d0-8d81-89df055de44e","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-9wtq2","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-13T16:09:36.429Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"650f0a91-efc1-46d0-8d81-89df055de44e"}
{"level":"error","ts":"2026-07-13T16:09:36.429Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"650f0a91-efc1-46d0-8d81-89df055de44e","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:09:36.429Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"71510b9a-f2e5-435c-b9c1-0a68162759ca"}
{"level":"info","ts":"2026-07-13T16:09:36.429Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"71510b9a-f2e5-435c-b9c1-0a68162759ca"}
{"level":"info","ts":"2026-07-13T16:09:36.429Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"71510b9a-f2e5-435c-b9c1-0a68162759ca","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:36.430Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"71510b9a-f2e5-435c-b9c1-0a68162759ca","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:36.430Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"71510b9a-f2e5-435c-b9c1-0a68162759ca","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:36.430Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"71510b9a-f2e5-435c-b9c1-0a68162759ca","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:09:36.430Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"71510b9a-f2e5-435c-b9c1-0a68162759ca","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:36.430Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"71510b9a-f2e5-435c-b9c1-0a68162759ca","worker Pod":"ray-cluster-small-group-worker-z7gbg","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-z7gbg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:36.430Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"71510b9a-f2e5-435c-b9c1-0a68162759ca","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:36.430Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"71510b9a-f2e5-435c-b9c1-0a68162759ca","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:36.430Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"71510b9a-f2e5-435c-b9c1-0a68162759ca","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:36.430Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"71510b9a-f2e5-435c-b9c1-0a68162759ca","podName":"ray-cluster-small-group-worker-tqz5h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:36.430Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"71510b9a-f2e5-435c-b9c1-0a68162759ca","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:36.430Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"71510b9a-f2e5-435c-b9c1-0a68162759ca","seconds":300}
{"level":"info","ts":"2026-07-13T16:09:36.435Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cbefdd3-d49b-4420-b018-fc5e75905a54"}
{"level":"info","ts":"2026-07-13T16:09:36.435Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cbefdd3-d49b-4420-b018-fc5e75905a54"}
{"level":"info","ts":"2026-07-13T16:09:36.435Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cbefdd3-d49b-4420-b018-fc5e75905a54","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:36.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cbefdd3-d49b-4420-b018-fc5e75905a54","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:36.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cbefdd3-d49b-4420-b018-fc5e75905a54","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:36.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cbefdd3-d49b-4420-b018-fc5e75905a54","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:09:36.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cbefdd3-d49b-4420-b018-fc5e75905a54","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:36.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cbefdd3-d49b-4420-b018-fc5e75905a54","worker Pod":"ray-cluster-small-group-worker-z7gbg","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-z7gbg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:36.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cbefdd3-d49b-4420-b018-fc5e75905a54","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:36.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cbefdd3-d49b-4420-b018-fc5e75905a54","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:36.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cbefdd3-d49b-4420-b018-fc5e75905a54","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:36.436Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cbefdd3-d49b-4420-b018-fc5e75905a54","podName":"ray-cluster-small-group-worker-tqz5h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:36.436Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cbefdd3-d49b-4420-b018-fc5e75905a54","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:36.436Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cbefdd3-d49b-4420-b018-fc5e75905a54","seconds":300}
{"level":"info","ts":"2026-07-13T16:09:36.439Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"fc6a4d73-7f7c-4951-b987-7da676a022cf","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:09:36.448Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"fc6a4d73-7f7c-4951-b987-7da676a022cf","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:09:36.448Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"fc6a4d73-7f7c-4951-b987-7da676a022cf","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:09:36.799Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d8258be-89fc-4201-8ad0-ba20721791ae"}
{"level":"info","ts":"2026-07-13T16:09:36.799Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d8258be-89fc-4201-8ad0-ba20721791ae"}
{"level":"info","ts":"2026-07-13T16:09:36.799Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d8258be-89fc-4201-8ad0-ba20721791ae","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:36.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d8258be-89fc-4201-8ad0-ba20721791ae","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:36.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d8258be-89fc-4201-8ad0-ba20721791ae","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:36.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d8258be-89fc-4201-8ad0-ba20721791ae","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:09:36.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d8258be-89fc-4201-8ad0-ba20721791ae","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:36.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d8258be-89fc-4201-8ad0-ba20721791ae","worker Pod":"ray-cluster-small-group-worker-z7gbg","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-z7gbg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:36.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d8258be-89fc-4201-8ad0-ba20721791ae","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:36.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d8258be-89fc-4201-8ad0-ba20721791ae","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:36.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d8258be-89fc-4201-8ad0-ba20721791ae","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:36.799Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d8258be-89fc-4201-8ad0-ba20721791ae","podName":"ray-cluster-small-group-worker-tqz5h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:36.799Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d8258be-89fc-4201-8ad0-ba20721791ae","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:36.799Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4d8258be-89fc-4201-8ad0-ba20721791ae","seconds":300}
{"level":"info","ts":"2026-07-13T16:09:37.842Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8446c472-86b8-47df-a449-dad7c9661737"}
{"level":"info","ts":"2026-07-13T16:09:37.842Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8446c472-86b8-47df-a449-dad7c9661737"}
{"level":"info","ts":"2026-07-13T16:09:37.842Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8446c472-86b8-47df-a449-dad7c9661737","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:09:37.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8446c472-86b8-47df-a449-dad7c9661737","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:09:37.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8446c472-86b8-47df-a449-dad7c9661737","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:37.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8446c472-86b8-47df-a449-dad7c9661737","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:09:37.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8446c472-86b8-47df-a449-dad7c9661737","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:37.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8446c472-86b8-47df-a449-dad7c9661737","worker Pod":"ray-cluster-small-group-worker-z7gbg","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-z7gbg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:09:37.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8446c472-86b8-47df-a449-dad7c9661737","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:37.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8446c472-86b8-47df-a449-dad7c9661737","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:09:37.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8446c472-86b8-47df-a449-dad7c9661737","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:09:37.842Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8446c472-86b8-47df-a449-dad7c9661737","podName":"ray-cluster-small-group-worker-tqz5h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:09:37.843Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8446c472-86b8-47df-a449-dad7c9661737","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:09:37.843Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8446c472-86b8-47df-a449-dad7c9661737","seconds":300}
{"level":"info","ts":"2026-07-13T16:10:06.347Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"e85731c2-3603-47fd-abfb-a07054692af7","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:10:06.347Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"e85731c2-3603-47fd-abfb-a07054692af7","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:10:06.347Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"e85731c2-3603-47fd-abfb-a07054692af7","pod":"ray-cluster-small-group-worker-tqz5h","ip":"10.134.0.37"}
{"level":"info","ts":"2026-07-13T16:10:06.347Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"e85731c2-3603-47fd-abfb-a07054692af7","pod":"ray-cluster-small-group-worker-z7gbg","ip":"10.134.0.38"}
{"level":"info","ts":"2026-07-13T16:10:06.361Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"e85731c2-3603-47fd-abfb-a07054692af7","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-9wtq2.svc","ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local"],"ipAddresses":["10.133.0.40","10.134.0.37","10.134.0.38","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:10:06.371Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"e85731c2-3603-47fd-abfb-a07054692af7","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-9wtq2.svc","ray-cluster-worker-svc.test-ns-9wtq2.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-9wtq2.svc","*.ray-cluster-worker-svc.test-ns-9wtq2.svc.cluster.local","*-worker-*.test-ns-9wtq2.svc","*-worker-*.test-ns-9wtq2.svc.cluster.local"],"ipAddresses":["10.133.0.40","10.134.0.37","10.134.0.38","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:10:06.371Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"e85731c2-3603-47fd-abfb-a07054692af7","rayCluster":"ray-cluster","podIPs":["10.133.0.40","10.134.0.37","10.134.0.38"]}
{"level":"info","ts":"2026-07-13T16:10:42.090Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2eee8855-5055-4ab1-9c6e-f02fddc52947"}
{"level":"info","ts":"2026-07-13T16:10:42.090Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2eee8855-5055-4ab1-9c6e-f02fddc52947"}
{"level":"info","ts":"2026-07-13T16:10:42.090Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2eee8855-5055-4ab1-9c6e-f02fddc52947","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:10:42.090Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2eee8855-5055-4ab1-9c6e-f02fddc52947","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:10:42.090Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2eee8855-5055-4ab1-9c6e-f02fddc52947","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:10:42.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2eee8855-5055-4ab1-9c6e-f02fddc52947","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:10:42.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2eee8855-5055-4ab1-9c6e-f02fddc52947","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:10:42.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2eee8855-5055-4ab1-9c6e-f02fddc52947","worker Pod":"ray-cluster-small-group-worker-z7gbg","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-z7gbg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:10:42.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2eee8855-5055-4ab1-9c6e-f02fddc52947","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:10:42.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2eee8855-5055-4ab1-9c6e-f02fddc52947","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:10:42.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2eee8855-5055-4ab1-9c6e-f02fddc52947","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:10:42.091Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2eee8855-5055-4ab1-9c6e-f02fddc52947","podName":"ray-cluster-small-group-worker-z7gbg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:10:42.091Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2eee8855-5055-4ab1-9c6e-f02fddc52947","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:10:42.091Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2eee8855-5055-4ab1-9c6e-f02fddc52947","seconds":300}
{"level":"info","ts":"2026-07-13T16:10:43.096Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0cda871a-7fcb-4993-8c44-292e1d9bd810"}
{"level":"info","ts":"2026-07-13T16:10:43.096Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0cda871a-7fcb-4993-8c44-292e1d9bd810"}
{"level":"info","ts":"2026-07-13T16:10:43.096Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0cda871a-7fcb-4993-8c44-292e1d9bd810","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:10:43.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0cda871a-7fcb-4993-8c44-292e1d9bd810","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:10:43.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0cda871a-7fcb-4993-8c44-292e1d9bd810","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:10:43.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0cda871a-7fcb-4993-8c44-292e1d9bd810","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:10:43.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0cda871a-7fcb-4993-8c44-292e1d9bd810","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:10:43.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0cda871a-7fcb-4993-8c44-292e1d9bd810","worker Pod":"ray-cluster-small-group-worker-z7gbg","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-z7gbg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:10:43.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0cda871a-7fcb-4993-8c44-292e1d9bd810","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:10:43.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0cda871a-7fcb-4993-8c44-292e1d9bd810","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:10:43.097Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0cda871a-7fcb-4993-8c44-292e1d9bd810","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:10:43.097Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0cda871a-7fcb-4993-8c44-292e1d9bd810","podName":"ray-cluster-small-group-worker-tqz5h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:10:43.097Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0cda871a-7fcb-4993-8c44-292e1d9bd810","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:10:43.097Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"0cda871a-7fcb-4993-8c44-292e1d9bd810","seconds":300}
{"level":"info","ts":"2026-07-13T16:10:53.134Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"450f99e7-c008-407f-9ee1-f8a946590551"}
{"level":"info","ts":"2026-07-13T16:10:53.134Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"450f99e7-c008-407f-9ee1-f8a946590551"}
{"level":"info","ts":"2026-07-13T16:10:53.134Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"450f99e7-c008-407f-9ee1-f8a946590551","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:10:53.135Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"450f99e7-c008-407f-9ee1-f8a946590551","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:10:53.135Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"450f99e7-c008-407f-9ee1-f8a946590551","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:10:53.135Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"450f99e7-c008-407f-9ee1-f8a946590551","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:10:53.135Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"450f99e7-c008-407f-9ee1-f8a946590551","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:10:53.135Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"450f99e7-c008-407f-9ee1-f8a946590551","worker Pod":"ray-cluster-small-group-worker-z7gbg","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-z7gbg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:10:53.135Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"450f99e7-c008-407f-9ee1-f8a946590551","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:10:53.135Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"450f99e7-c008-407f-9ee1-f8a946590551","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:10:53.135Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"450f99e7-c008-407f-9ee1-f8a946590551","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:10:53.135Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"450f99e7-c008-407f-9ee1-f8a946590551","podName":"ray-cluster-small-group-worker-tqz5h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:10:53.135Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"450f99e7-c008-407f-9ee1-f8a946590551","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:10:53.135Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"450f99e7-c008-407f-9ee1-f8a946590551","seconds":300}
{"level":"info","ts":"2026-07-13T16:10:54.142Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ec1d555f-3522-4efe-9ec6-dd90a14e3596"}
{"level":"info","ts":"2026-07-13T16:10:54.142Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ec1d555f-3522-4efe-9ec6-dd90a14e3596"}
{"level":"info","ts":"2026-07-13T16:10:54.142Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ec1d555f-3522-4efe-9ec6-dd90a14e3596","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:10:54.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ec1d555f-3522-4efe-9ec6-dd90a14e3596","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:10:54.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ec1d555f-3522-4efe-9ec6-dd90a14e3596","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:10:54.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ec1d555f-3522-4efe-9ec6-dd90a14e3596","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:10:54.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ec1d555f-3522-4efe-9ec6-dd90a14e3596","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:10:54.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ec1d555f-3522-4efe-9ec6-dd90a14e3596","worker Pod":"ray-cluster-small-group-worker-z7gbg","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-z7gbg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:10:54.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ec1d555f-3522-4efe-9ec6-dd90a14e3596","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:10:54.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ec1d555f-3522-4efe-9ec6-dd90a14e3596","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:10:54.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ec1d555f-3522-4efe-9ec6-dd90a14e3596","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:10:54.142Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ec1d555f-3522-4efe-9ec6-dd90a14e3596","podName":"ray-cluster-small-group-worker-tqz5h","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:09:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:10:54.142Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ec1d555f-3522-4efe-9ec6-dd90a14e3596","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:10:54.142Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ec1d555f-3522-4efe-9ec6-dd90a14e3596","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:09:36Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:10:54Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-13T16:10:54.154Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"16e79df1-c9e0-4b0b-965c-3be2ac0b2f45","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:10:54.154Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"3f4ead76-dc1e-41f1-946e-2e388e1e74ec","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:10:54.154Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"16e79df1-c9e0-4b0b-965c-3be2ac0b2f45","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:10:54.154Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"16e79df1-c9e0-4b0b-965c-3be2ac0b2f45","pod":"ray-cluster-small-group-worker-tqz5h","ip":"10.134.0.37"}
{"level":"info","ts":"2026-07-13T16:10:54.154Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"16e79df1-c9e0-4b0b-965c-3be2ac0b2f45","pod":"ray-cluster-small-group-worker-z7gbg","ip":"10.134.0.38"}
{"level":"info","ts":"2026-07-13T16:10:54.154Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"16e79df1-c9e0-4b0b-965c-3be2ac0b2f45","rayCluster":"ray-cluster","podIPs":["10.133.0.40","10.134.0.37","10.134.0.38"]}
{"level":"info","ts":"2026-07-13T16:10:54.169Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"3f4ead76-dc1e-41f1-946e-2e388e1e74ec","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:10:54.177Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"3f4ead76-dc1e-41f1-946e-2e388e1e74ec","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:10:54.177Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"3f4ead76-dc1e-41f1-946e-2e388e1e74ec","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:10:56.154Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"096d6ab2-a46a-41b2-8011-77f8ebae2490"}
{"level":"info","ts":"2026-07-13T16:10:56.154Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"096d6ab2-a46a-41b2-8011-77f8ebae2490"}
{"level":"info","ts":"2026-07-13T16:10:56.154Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"096d6ab2-a46a-41b2-8011-77f8ebae2490","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:10:56.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"096d6ab2-a46a-41b2-8011-77f8ebae2490","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:10:56.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"096d6ab2-a46a-41b2-8011-77f8ebae2490","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:10:56.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"096d6ab2-a46a-41b2-8011-77f8ebae2490","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:10:56.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"096d6ab2-a46a-41b2-8011-77f8ebae2490","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:10:56.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"096d6ab2-a46a-41b2-8011-77f8ebae2490","worker Pod":"ray-cluster-small-group-worker-z7gbg","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-z7gbg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:10:56.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"096d6ab2-a46a-41b2-8011-77f8ebae2490","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:10:56.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"096d6ab2-a46a-41b2-8011-77f8ebae2490","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:10:56.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"096d6ab2-a46a-41b2-8011-77f8ebae2490","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:10:56.154Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"096d6ab2-a46a-41b2-8011-77f8ebae2490","podName":"ray-cluster-small-group-worker-tqz5h","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:09:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:10:56.155Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"096d6ab2-a46a-41b2-8011-77f8ebae2490","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:10:56.155Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"096d6ab2-a46a-41b2-8011-77f8ebae2490","seconds":300}
{"level":"info","ts":"2026-07-13T16:10:58.158Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a2d36a41-7cca-4ec0-ab71-e493593103c0"}
{"level":"info","ts":"2026-07-13T16:10:58.158Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a2d36a41-7cca-4ec0-ab71-e493593103c0"}
{"level":"info","ts":"2026-07-13T16:10:58.159Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a2d36a41-7cca-4ec0-ab71-e493593103c0","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:10:58.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a2d36a41-7cca-4ec0-ab71-e493593103c0","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:10:58.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a2d36a41-7cca-4ec0-ab71-e493593103c0","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:10:58.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a2d36a41-7cca-4ec0-ab71-e493593103c0","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:10:58.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a2d36a41-7cca-4ec0-ab71-e493593103c0","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:10:58.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a2d36a41-7cca-4ec0-ab71-e493593103c0","worker Pod":"ray-cluster-small-group-worker-z7gbg","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-z7gbg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:10:58.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a2d36a41-7cca-4ec0-ab71-e493593103c0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:10:58.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a2d36a41-7cca-4ec0-ab71-e493593103c0","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:10:58.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a2d36a41-7cca-4ec0-ab71-e493593103c0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:10:58.159Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a2d36a41-7cca-4ec0-ab71-e493593103c0","podName":"ray-cluster-small-group-worker-tqz5h","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:09:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:10:58.159Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a2d36a41-7cca-4ec0-ab71-e493593103c0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:10:58.159Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a2d36a41-7cca-4ec0-ab71-e493593103c0","seconds":300}
{"level":"info","ts":"2026-07-13T16:10:59.187Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1267cb6c-48ce-48c6-b626-8c23ae91a870"}
{"level":"info","ts":"2026-07-13T16:10:59.187Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1267cb6c-48ce-48c6-b626-8c23ae91a870"}
{"level":"info","ts":"2026-07-13T16:10:59.187Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1267cb6c-48ce-48c6-b626-8c23ae91a870","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:10:59.187Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1267cb6c-48ce-48c6-b626-8c23ae91a870","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:10:59.187Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1267cb6c-48ce-48c6-b626-8c23ae91a870","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:10:59.187Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1267cb6c-48ce-48c6-b626-8c23ae91a870","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:10:59.187Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1267cb6c-48ce-48c6-b626-8c23ae91a870","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:10:59.187Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1267cb6c-48ce-48c6-b626-8c23ae91a870","worker Pod":"ray-cluster-small-group-worker-z7gbg","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-z7gbg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:10:59.187Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1267cb6c-48ce-48c6-b626-8c23ae91a870","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:10:59.187Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1267cb6c-48ce-48c6-b626-8c23ae91a870","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:10:59.187Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1267cb6c-48ce-48c6-b626-8c23ae91a870","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:10:59.188Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1267cb6c-48ce-48c6-b626-8c23ae91a870","podName":"ray-cluster-small-group-worker-tqz5h","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:09:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:10:59.188Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1267cb6c-48ce-48c6-b626-8c23ae91a870","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:10:59.188Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1267cb6c-48ce-48c6-b626-8c23ae91a870","seconds":300}
{"level":"info","ts":"2026-07-13T16:11:05.163Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ae75dd56-c39d-4a73-b70d-276244541a74"}
{"level":"info","ts":"2026-07-13T16:11:05.163Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ae75dd56-c39d-4a73-b70d-276244541a74"}
{"level":"info","ts":"2026-07-13T16:11:05.163Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ae75dd56-c39d-4a73-b70d-276244541a74","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:11:05.163Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ae75dd56-c39d-4a73-b70d-276244541a74","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:11:05.163Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ae75dd56-c39d-4a73-b70d-276244541a74","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:11:05.163Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ae75dd56-c39d-4a73-b70d-276244541a74","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:11:05.163Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ae75dd56-c39d-4a73-b70d-276244541a74","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:11:05.163Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ae75dd56-c39d-4a73-b70d-276244541a74","worker Pod":"ray-cluster-small-group-worker-z7gbg","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-z7gbg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:11:05.163Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ae75dd56-c39d-4a73-b70d-276244541a74","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:11:05.164Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ae75dd56-c39d-4a73-b70d-276244541a74","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:11:05.164Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ae75dd56-c39d-4a73-b70d-276244541a74","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:11:05.164Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ae75dd56-c39d-4a73-b70d-276244541a74","podName":"ray-cluster-small-group-worker-z7gbg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:11:05.164Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ae75dd56-c39d-4a73-b70d-276244541a74","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:11:05.164Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ae75dd56-c39d-4a73-b70d-276244541a74","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:10:54Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:11:05Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-13T16:11:05.175Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"8b399170-bc7e-4908-b258-106884a69e27","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:11:05.176Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"f3550591-ecba-4776-a5b8-b163ea4a3ab4","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:11:05.176Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"8b399170-bc7e-4908-b258-106884a69e27","pod":"ray-cluster-small-group-worker-tqz5h","ip":"10.134.0.37"}
{"level":"info","ts":"2026-07-13T16:11:05.176Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"8b399170-bc7e-4908-b258-106884a69e27","pod":"ray-cluster-small-group-worker-z7gbg","ip":"10.134.0.38"}
{"level":"info","ts":"2026-07-13T16:11:05.176Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"8b399170-bc7e-4908-b258-106884a69e27","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:11:05.176Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"8b399170-bc7e-4908-b258-106884a69e27","rayCluster":"ray-cluster","podIPs":["10.134.0.37","10.134.0.38","10.133.0.40"]}
{"level":"info","ts":"2026-07-13T16:11:05.187Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"f3550591-ecba-4776-a5b8-b163ea4a3ab4","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:11:05.195Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"f3550591-ecba-4776-a5b8-b163ea4a3ab4","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:11:05.195Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"f3550591-ecba-4776-a5b8-b163ea4a3ab4","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:11:06.372Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"69b0fb34-1c35-40cf-a891-2c0aa48a619e","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:11:06.372Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"69b0fb34-1c35-40cf-a891-2c0aa48a619e","pod":"ray-cluster-small-group-worker-z7gbg","ip":"10.134.0.38"}
{"level":"info","ts":"2026-07-13T16:11:06.372Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"69b0fb34-1c35-40cf-a891-2c0aa48a619e","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:11:06.372Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"69b0fb34-1c35-40cf-a891-2c0aa48a619e","pod":"ray-cluster-small-group-worker-tqz5h","ip":"10.134.0.37"}
{"level":"info","ts":"2026-07-13T16:11:06.372Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"69b0fb34-1c35-40cf-a891-2c0aa48a619e","rayCluster":"ray-cluster","podIPs":["10.134.0.38","10.133.0.40","10.134.0.37"]}
{"level":"info","ts":"2026-07-13T16:11:07.176Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"75f1e555-a4d0-4817-b227-127527a9bf11"}
{"level":"info","ts":"2026-07-13T16:11:07.176Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"75f1e555-a4d0-4817-b227-127527a9bf11"}
{"level":"info","ts":"2026-07-13T16:11:07.176Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"75f1e555-a4d0-4817-b227-127527a9bf11","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:11:07.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"75f1e555-a4d0-4817-b227-127527a9bf11","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:11:07.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"75f1e555-a4d0-4817-b227-127527a9bf11","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:11:07.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"75f1e555-a4d0-4817-b227-127527a9bf11","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:11:07.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"75f1e555-a4d0-4817-b227-127527a9bf11","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:11:07.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"75f1e555-a4d0-4817-b227-127527a9bf11","worker Pod":"ray-cluster-small-group-worker-z7gbg","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-z7gbg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:11:07.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"75f1e555-a4d0-4817-b227-127527a9bf11","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:11:07.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"75f1e555-a4d0-4817-b227-127527a9bf11","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:11:07.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"75f1e555-a4d0-4817-b227-127527a9bf11","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:11:07.176Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"75f1e555-a4d0-4817-b227-127527a9bf11","podName":"ray-cluster-small-group-worker-z7gbg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:11:07.176Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"75f1e555-a4d0-4817-b227-127527a9bf11","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:11:07.176Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"75f1e555-a4d0-4817-b227-127527a9bf11","seconds":300}
{"level":"info","ts":"2026-07-13T16:11:08.200Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1201a9ca-a03b-4de9-b868-21015c3dc5dd"}
{"level":"info","ts":"2026-07-13T16:11:08.200Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1201a9ca-a03b-4de9-b868-21015c3dc5dd"}
{"level":"info","ts":"2026-07-13T16:11:08.200Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1201a9ca-a03b-4de9-b868-21015c3dc5dd","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:11:08.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1201a9ca-a03b-4de9-b868-21015c3dc5dd","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:11:08.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1201a9ca-a03b-4de9-b868-21015c3dc5dd","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:11:08.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1201a9ca-a03b-4de9-b868-21015c3dc5dd","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:11:08.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1201a9ca-a03b-4de9-b868-21015c3dc5dd","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:11:08.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1201a9ca-a03b-4de9-b868-21015c3dc5dd","worker Pod":"ray-cluster-small-group-worker-z7gbg","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-z7gbg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:11:08.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1201a9ca-a03b-4de9-b868-21015c3dc5dd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:11:08.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1201a9ca-a03b-4de9-b868-21015c3dc5dd","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:11:08.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1201a9ca-a03b-4de9-b868-21015c3dc5dd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:11:08.201Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1201a9ca-a03b-4de9-b868-21015c3dc5dd","podName":"ray-cluster-small-group-worker-z7gbg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:11:08.201Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1201a9ca-a03b-4de9-b868-21015c3dc5dd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:11:08.201Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1201a9ca-a03b-4de9-b868-21015c3dc5dd","seconds":300}
{"level":"info","ts":"2026-07-13T16:11:09.208Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"297e2fcc-6739-4e50-92ce-e103e1232b07"}
{"level":"info","ts":"2026-07-13T16:11:09.209Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"297e2fcc-6739-4e50-92ce-e103e1232b07"}
{"level":"info","ts":"2026-07-13T16:11:09.209Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"297e2fcc-6739-4e50-92ce-e103e1232b07","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:11:09.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"297e2fcc-6739-4e50-92ce-e103e1232b07","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:11:09.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"297e2fcc-6739-4e50-92ce-e103e1232b07","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:11:09.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"297e2fcc-6739-4e50-92ce-e103e1232b07","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:11:09.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"297e2fcc-6739-4e50-92ce-e103e1232b07","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:11:09.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"297e2fcc-6739-4e50-92ce-e103e1232b07","worker Pod":"ray-cluster-small-group-worker-z7gbg","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-z7gbg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:11:09.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"297e2fcc-6739-4e50-92ce-e103e1232b07","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:11:09.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"297e2fcc-6739-4e50-92ce-e103e1232b07","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:11:09.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"297e2fcc-6739-4e50-92ce-e103e1232b07","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:11:09.209Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"297e2fcc-6739-4e50-92ce-e103e1232b07","podName":"ray-cluster-small-group-worker-z7gbg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:09:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:11:09.209Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"297e2fcc-6739-4e50-92ce-e103e1232b07","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:11:09.209Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"297e2fcc-6739-4e50-92ce-e103e1232b07","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:11:05Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:11:09Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-13T16:11:09.222Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"8b81abdc-74d0-4580-b643-6103a04b8471","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:11:09.222Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"05ca77ea-a4ed-4aae-bfcd-1347631b0f6e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:11:09.222Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"8b81abdc-74d0-4580-b643-6103a04b8471","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:11:09.222Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"8b81abdc-74d0-4580-b643-6103a04b8471","pod":"ray-cluster-small-group-worker-tqz5h","ip":"10.134.0.37"}
{"level":"info","ts":"2026-07-13T16:11:09.222Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"8b81abdc-74d0-4580-b643-6103a04b8471","pod":"ray-cluster-small-group-worker-z7gbg","ip":"10.134.0.38"}
{"level":"info","ts":"2026-07-13T16:11:09.222Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"8b81abdc-74d0-4580-b643-6103a04b8471","rayCluster":"ray-cluster","podIPs":["10.133.0.40","10.134.0.37","10.134.0.38"]}
{"level":"info","ts":"2026-07-13T16:11:09.237Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"05ca77ea-a4ed-4aae-bfcd-1347631b0f6e","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:11:09.245Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"05ca77ea-a4ed-4aae-bfcd-1347631b0f6e","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:11:09.245Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"05ca77ea-a4ed-4aae-bfcd-1347631b0f6e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:11:11.222Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b948d5f3-2b5f-4efc-92ec-2d5bd438977e"}
{"level":"info","ts":"2026-07-13T16:11:11.222Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b948d5f3-2b5f-4efc-92ec-2d5bd438977e"}
{"level":"info","ts":"2026-07-13T16:11:11.222Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b948d5f3-2b5f-4efc-92ec-2d5bd438977e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:11:11.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b948d5f3-2b5f-4efc-92ec-2d5bd438977e","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:11:11.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b948d5f3-2b5f-4efc-92ec-2d5bd438977e","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:11:11.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b948d5f3-2b5f-4efc-92ec-2d5bd438977e","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:11:11.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b948d5f3-2b5f-4efc-92ec-2d5bd438977e","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:11:11.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b948d5f3-2b5f-4efc-92ec-2d5bd438977e","worker Pod":"ray-cluster-small-group-worker-z7gbg","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-z7gbg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:11:11.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b948d5f3-2b5f-4efc-92ec-2d5bd438977e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:11:11.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b948d5f3-2b5f-4efc-92ec-2d5bd438977e","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:11:11.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b948d5f3-2b5f-4efc-92ec-2d5bd438977e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:11:11.222Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b948d5f3-2b5f-4efc-92ec-2d5bd438977e","podName":"ray-cluster-small-group-worker-z7gbg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:09:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:11:11.222Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b948d5f3-2b5f-4efc-92ec-2d5bd438977e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:11:11.222Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b948d5f3-2b5f-4efc-92ec-2d5bd438977e","seconds":300}
{"level":"info","ts":"2026-07-13T16:11:20.232Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"14e1557a-5604-4905-9078-53d8e9ebd54c"}
{"level":"info","ts":"2026-07-13T16:11:20.232Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"14e1557a-5604-4905-9078-53d8e9ebd54c"}
{"level":"info","ts":"2026-07-13T16:11:20.232Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"14e1557a-5604-4905-9078-53d8e9ebd54c","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:11:20.232Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"14e1557a-5604-4905-9078-53d8e9ebd54c","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:11:20.232Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"14e1557a-5604-4905-9078-53d8e9ebd54c","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:11:20.232Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"14e1557a-5604-4905-9078-53d8e9ebd54c","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:11:20.232Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"14e1557a-5604-4905-9078-53d8e9ebd54c","worker Pod":"ray-cluster-small-group-worker-z7gbg","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-z7gbg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:11:20.232Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"14e1557a-5604-4905-9078-53d8e9ebd54c","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:11:20.232Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"14e1557a-5604-4905-9078-53d8e9ebd54c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:11:20.232Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"14e1557a-5604-4905-9078-53d8e9ebd54c","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:11:20.232Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"14e1557a-5604-4905-9078-53d8e9ebd54c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:11:20.232Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"14e1557a-5604-4905-9078-53d8e9ebd54c","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":2,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:11:20.232Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"14e1557a-5604-4905-9078-53d8e9ebd54c","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:11:09Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:11:20Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-13T16:11:20.244Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"1fd70e65-9394-4b10-b61a-b19e9e379271","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:11:20.244Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"2eccd78e-fb3a-4d27-aba0-a7e2143d65e0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:11:20.244Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"1fd70e65-9394-4b10-b61a-b19e9e379271","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:11:20.244Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"1fd70e65-9394-4b10-b61a-b19e9e379271","pod":"ray-cluster-small-group-worker-tqz5h","ip":"10.134.0.37"}
{"level":"info","ts":"2026-07-13T16:11:20.244Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"1fd70e65-9394-4b10-b61a-b19e9e379271","pod":"ray-cluster-small-group-worker-z7gbg","ip":"10.134.0.38"}
{"level":"info","ts":"2026-07-13T16:11:20.244Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"1fd70e65-9394-4b10-b61a-b19e9e379271","rayCluster":"ray-cluster","podIPs":["10.133.0.40","10.134.0.37","10.134.0.38"]}
{"level":"info","ts":"2026-07-13T16:11:20.252Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"2eccd78e-fb3a-4d27-aba0-a7e2143d65e0","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:11:20.259Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"2eccd78e-fb3a-4d27-aba0-a7e2143d65e0","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:11:20.259Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"2eccd78e-fb3a-4d27-aba0-a7e2143d65e0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:11:22.244Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1898fa73-b420-4155-96d6-fc928ef0d49d"}
{"level":"info","ts":"2026-07-13T16:11:22.244Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1898fa73-b420-4155-96d6-fc928ef0d49d"}
{"level":"info","ts":"2026-07-13T16:11:22.244Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1898fa73-b420-4155-96d6-fc928ef0d49d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:11:22.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1898fa73-b420-4155-96d6-fc928ef0d49d","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:11:22.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1898fa73-b420-4155-96d6-fc928ef0d49d","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:11:22.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1898fa73-b420-4155-96d6-fc928ef0d49d","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:11:22.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1898fa73-b420-4155-96d6-fc928ef0d49d","worker Pod":"ray-cluster-small-group-worker-z7gbg","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-z7gbg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:11:22.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1898fa73-b420-4155-96d6-fc928ef0d49d","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:11:22.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1898fa73-b420-4155-96d6-fc928ef0d49d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:11:22.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1898fa73-b420-4155-96d6-fc928ef0d49d","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:11:22.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1898fa73-b420-4155-96d6-fc928ef0d49d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:11:22.245Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1898fa73-b420-4155-96d6-fc928ef0d49d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:11:22.245Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1898fa73-b420-4155-96d6-fc928ef0d49d","seconds":300}
{"level":"info","ts":"2026-07-13T16:12:06.373Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0e01b920-c2f1-4d6c-a818-64afd04578ba","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:12:06.373Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0e01b920-c2f1-4d6c-a818-64afd04578ba","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:12:06.373Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0e01b920-c2f1-4d6c-a818-64afd04578ba","pod":"ray-cluster-small-group-worker-tqz5h","ip":"10.134.0.37"}
{"level":"info","ts":"2026-07-13T16:12:06.373Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0e01b920-c2f1-4d6c-a818-64afd04578ba","pod":"ray-cluster-small-group-worker-z7gbg","ip":"10.134.0.38"}
{"level":"info","ts":"2026-07-13T16:12:06.373Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"0e01b920-c2f1-4d6c-a818-64afd04578ba","rayCluster":"ray-cluster","podIPs":["10.133.0.40","10.134.0.37","10.134.0.38"]}
{"level":"info","ts":"2026-07-13T16:12:18.394Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:12:18.394Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-9wtq2"}
{"level":"info","ts":"2026-07-13T16:12:18.404Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"3bfbc81f-81ac-4844-a3b2-ed5af5c6fe83","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:12:18.404Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a2bffea8-0be5-4832-8a82-57862cde2a8b","namespacedName":{"name":"ray-cluster","namespace":"test-ns-9wtq2"}}
{"level":"info","ts":"2026-07-13T16:12:18.404Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d7671966-428f-4c19-ac87-6f4b98bc2048","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:12:18.404Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a2bffea8-0be5-4832-8a82-57862cde2a8b","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:12:18.404Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a2bffea8-0be5-4832-8a82-57862cde2a8b","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:12:18.404Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1990ddc5-f104-43f1-b56d-dda938344d44"}
{"level":"info","ts":"2026-07-13T16:12:18.404Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1990ddc5-f104-43f1-b56d-dda938344d44"}
{"level":"info","ts":"2026-07-13T16:12:18.404Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"3bfbc81f-81ac-4844-a3b2-ed5af5c6fe83","pod":"ray-cluster-small-group-worker-z7gbg","ip":"10.134.0.38"}
{"level":"info","ts":"2026-07-13T16:12:18.404Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"3bfbc81f-81ac-4844-a3b2-ed5af5c6fe83","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:12:18.404Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"3bfbc81f-81ac-4844-a3b2-ed5af5c6fe83","pod":"ray-cluster-small-group-worker-tqz5h","ip":"10.134.0.37"}
{"level":"info","ts":"2026-07-13T16:12:18.404Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1990ddc5-f104-43f1-b56d-dda938344d44","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:12:18.405Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"3bfbc81f-81ac-4844-a3b2-ed5af5c6fe83","rayCluster":"ray-cluster","podIPs":["10.134.0.38","10.133.0.40","10.134.0.37"]}
{"level":"info","ts":"2026-07-13T16:12:18.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1990ddc5-f104-43f1-b56d-dda938344d44","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:12:18.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1990ddc5-f104-43f1-b56d-dda938344d44","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:12:18.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1990ddc5-f104-43f1-b56d-dda938344d44","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-13T16:12:18.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1990ddc5-f104-43f1-b56d-dda938344d44","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:12:18.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1990ddc5-f104-43f1-b56d-dda938344d44","worker Pod":"ray-cluster-small-group-worker-z7gbg","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-z7gbg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:12:18.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1990ddc5-f104-43f1-b56d-dda938344d44","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:12:18.405Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1990ddc5-f104-43f1-b56d-dda938344d44","namespace":"test-ns-9wtq2","name":"ray-cluster-small-group-worker-z7gbg"}
{"level":"info","ts":"2026-07-13T16:12:18.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1990ddc5-f104-43f1-b56d-dda938344d44","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:12:18.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1990ddc5-f104-43f1-b56d-dda938344d44","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:12:18.414Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1990ddc5-f104-43f1-b56d-dda938344d44","oldReadyWorkerReplicas":2,"newReadyWorkerReplicas":2,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:12:18.414Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1990ddc5-f104-43f1-b56d-dda938344d44","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:11:20Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:12:18Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-07-13T16:12:18.417Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a2bffea8-0be5-4832-8a82-57862cde2a8b","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:12:18.417Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"58604d2d-42ce-474d-b786-a4fc1d8fa483","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:12:18.417Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"58604d2d-42ce-474d-b786-a4fc1d8fa483","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:12:18.417Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"58604d2d-42ce-474d-b786-a4fc1d8fa483","pod":"ray-cluster-small-group-worker-tqz5h","ip":"10.134.0.37"}
{"level":"info","ts":"2026-07-13T16:12:18.417Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"58604d2d-42ce-474d-b786-a4fc1d8fa483","pod":"ray-cluster-small-group-worker-z7gbg","ip":"10.134.0.38"}
{"level":"info","ts":"2026-07-13T16:12:18.417Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"58604d2d-42ce-474d-b786-a4fc1d8fa483","rayCluster":"ray-cluster","podIPs":["10.133.0.40","10.134.0.37","10.134.0.38"]}
{"level":"info","ts":"2026-07-13T16:12:18.422Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1990ddc5-f104-43f1-b56d-dda938344d44","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-9wtq2","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-13T16:12:18.422Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1990ddc5-f104-43f1-b56d-dda938344d44"}
{"level":"error","ts":"2026-07-13T16:12:18.422Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"1990ddc5-f104-43f1-b56d-dda938344d44","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:12:18.422Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8b929c2f-d5ce-470a-9462-45a42fcf3166"}
{"level":"info","ts":"2026-07-13T16:12:18.422Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8b929c2f-d5ce-470a-9462-45a42fcf3166"}
{"level":"info","ts":"2026-07-13T16:12:18.423Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8b929c2f-d5ce-470a-9462-45a42fcf3166","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:12:18.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8b929c2f-d5ce-470a-9462-45a42fcf3166","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:12:18.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8b929c2f-d5ce-470a-9462-45a42fcf3166","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:12:18.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8b929c2f-d5ce-470a-9462-45a42fcf3166","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:12:18.423Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8b929c2f-d5ce-470a-9462-45a42fcf3166","oldReadyWorkerReplicas":2,"newReadyWorkerReplicas":2,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:12:18.423Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"8b929c2f-d5ce-470a-9462-45a42fcf3166","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:11:20Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:12:18Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-07-13T16:12:18.425Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d7671966-428f-4c19-ac87-6f4b98bc2048","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:12:18.433Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5c18df94-f14a-49ca-9e28-20937606ab08"}
{"level":"info","ts":"2026-07-13T16:12:18.433Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5c18df94-f14a-49ca-9e28-20937606ab08"}
{"level":"info","ts":"2026-07-13T16:12:18.433Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5c18df94-f14a-49ca-9e28-20937606ab08","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:12:18.434Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5c18df94-f14a-49ca-9e28-20937606ab08","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:12:18.434Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5c18df94-f14a-49ca-9e28-20937606ab08","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:12:18.434Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5c18df94-f14a-49ca-9e28-20937606ab08","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:12:18.434Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"fc55e547-9bd9-4c7a-ae7a-1c93f64dc281","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:12:18.434Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5c18df94-f14a-49ca-9e28-20937606ab08","oldReadyWorkerReplicas":2,"newReadyWorkerReplicas":2,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:12:18.434Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"fc55e547-9bd9-4c7a-ae7a-1c93f64dc281","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:12:18.434Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"fc55e547-9bd9-4c7a-ae7a-1c93f64dc281","pod":"ray-cluster-small-group-worker-tqz5h","ip":"10.134.0.37"}
{"level":"info","ts":"2026-07-13T16:12:18.434Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"fc55e547-9bd9-4c7a-ae7a-1c93f64dc281","pod":"ray-cluster-small-group-worker-z7gbg","ip":"10.134.0.38"}
{"level":"info","ts":"2026-07-13T16:12:18.434Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5c18df94-f14a-49ca-9e28-20937606ab08","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:11:20Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:12:18Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-07-13T16:12:18.434Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"fc55e547-9bd9-4c7a-ae7a-1c93f64dc281","rayCluster":"ray-cluster","podIPs":["10.133.0.40","10.134.0.37","10.134.0.38"]}
{"level":"info","ts":"2026-07-13T16:12:18.437Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d7671966-428f-4c19-ac87-6f4b98bc2048","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:12:18.437Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d7671966-428f-4c19-ac87-6f4b98bc2048","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:12:18.437Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d7f36087-594f-41ba-97df-1cde1a7aaca6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:12:18.440Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5c18df94-f14a-49ca-9e28-20937606ab08","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-9wtq2","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-13T16:12:18.440Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5c18df94-f14a-49ca-9e28-20937606ab08"}
{"level":"error","ts":"2026-07-13T16:12:18.440Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5c18df94-f14a-49ca-9e28-20937606ab08","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:12:18.440Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b8d9d51d-60cf-418e-81c5-4928f9feea9c"}
{"level":"info","ts":"2026-07-13T16:12:18.440Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b8d9d51d-60cf-418e-81c5-4928f9feea9c"}
{"level":"info","ts":"2026-07-13T16:12:18.440Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b8d9d51d-60cf-418e-81c5-4928f9feea9c","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:12:18.440Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b8d9d51d-60cf-418e-81c5-4928f9feea9c","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:12:18.440Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b8d9d51d-60cf-418e-81c5-4928f9feea9c","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:12:18.440Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b8d9d51d-60cf-418e-81c5-4928f9feea9c","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:12:18.440Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b8d9d51d-60cf-418e-81c5-4928f9feea9c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:12:18.440Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"b8d9d51d-60cf-418e-81c5-4928f9feea9c","seconds":300}
{"level":"info","ts":"2026-07-13T16:12:18.446Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"616da905-af08-40e5-8bb4-dfcd2346e2e5"}
{"level":"info","ts":"2026-07-13T16:12:18.446Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"616da905-af08-40e5-8bb4-dfcd2346e2e5"}
{"level":"info","ts":"2026-07-13T16:12:18.446Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"616da905-af08-40e5-8bb4-dfcd2346e2e5","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:12:18.446Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"616da905-af08-40e5-8bb4-dfcd2346e2e5","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:12:18.446Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"616da905-af08-40e5-8bb4-dfcd2346e2e5","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:12:18.446Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"616da905-af08-40e5-8bb4-dfcd2346e2e5","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:12:18.446Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"616da905-af08-40e5-8bb4-dfcd2346e2e5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:12:18.446Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"616da905-af08-40e5-8bb4-dfcd2346e2e5","seconds":300}
{"level":"info","ts":"2026-07-13T16:12:18.448Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d7f36087-594f-41ba-97df-1cde1a7aaca6","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:12:18.455Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d7f36087-594f-41ba-97df-1cde1a7aaca6","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:12:18.455Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d7f36087-594f-41ba-97df-1cde1a7aaca6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:12:18.978Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ca4bb684-0b02-40a1-9b3f-eca171967f21"}
{"level":"info","ts":"2026-07-13T16:12:18.978Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ca4bb684-0b02-40a1-9b3f-eca171967f21"}
{"level":"info","ts":"2026-07-13T16:12:18.978Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ca4bb684-0b02-40a1-9b3f-eca171967f21","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:12:18.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ca4bb684-0b02-40a1-9b3f-eca171967f21","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:12:18.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ca4bb684-0b02-40a1-9b3f-eca171967f21","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:12:18.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ca4bb684-0b02-40a1-9b3f-eca171967f21","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:12:18.978Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ca4bb684-0b02-40a1-9b3f-eca171967f21","oldReadyWorkerReplicas":2,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:12:18.978Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ca4bb684-0b02-40a1-9b3f-eca171967f21","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:12:18Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:12:18Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-07-13T16:12:18.990Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"7cc73b0b-79b0-4a99-9cf1-07d58ba26185","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:12:18.990Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d46caeb3-b116-4bbc-b7c2-f15e00968c18","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:12:18.990Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"7cc73b0b-79b0-4a99-9cf1-07d58ba26185","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:12:18.990Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"7cc73b0b-79b0-4a99-9cf1-07d58ba26185","pod":"ray-cluster-small-group-worker-tqz5h","ip":"10.134.0.37"}
{"level":"info","ts":"2026-07-13T16:12:18.990Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"7cc73b0b-79b0-4a99-9cf1-07d58ba26185","pod":"ray-cluster-small-group-worker-z7gbg","ip":"10.134.0.38"}
{"level":"info","ts":"2026-07-13T16:12:18.990Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"7cc73b0b-79b0-4a99-9cf1-07d58ba26185","rayCluster":"ray-cluster","podIPs":["10.133.0.40","10.134.0.37","10.134.0.38"]}
{"level":"info","ts":"2026-07-13T16:12:18.999Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d46caeb3-b116-4bbc-b7c2-f15e00968c18","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:12:19.006Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d46caeb3-b116-4bbc-b7c2-f15e00968c18","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:12:19.006Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d46caeb3-b116-4bbc-b7c2-f15e00968c18","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:12:19.485Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5662af1e-96d1-43f9-b0f0-d38d46aa110f"}
{"level":"info","ts":"2026-07-13T16:12:19.485Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5662af1e-96d1-43f9-b0f0-d38d46aa110f"}
{"level":"info","ts":"2026-07-13T16:12:19.485Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5662af1e-96d1-43f9-b0f0-d38d46aa110f","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:12:19.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5662af1e-96d1-43f9-b0f0-d38d46aa110f","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:12:19.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5662af1e-96d1-43f9-b0f0-d38d46aa110f","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:12:19.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5662af1e-96d1-43f9-b0f0-d38d46aa110f","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:12:19.485Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5662af1e-96d1-43f9-b0f0-d38d46aa110f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:12:19.486Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5662af1e-96d1-43f9-b0f0-d38d46aa110f","seconds":300}
{"level":"info","ts":"2026-07-13T16:12:19.488Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"796a6b13-4379-4833-85d0-51e55f122c51"}
{"level":"info","ts":"2026-07-13T16:12:19.488Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"796a6b13-4379-4833-85d0-51e55f122c51"}
{"level":"info","ts":"2026-07-13T16:12:19.488Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"796a6b13-4379-4833-85d0-51e55f122c51","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:12:19.488Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"796a6b13-4379-4833-85d0-51e55f122c51","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:12:19.488Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"796a6b13-4379-4833-85d0-51e55f122c51","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:12:19.488Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"796a6b13-4379-4833-85d0-51e55f122c51","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-13T16:12:19.488Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"796a6b13-4379-4833-85d0-51e55f122c51","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:12:19.488Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"796a6b13-4379-4833-85d0-51e55f122c51","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:12:19.488Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"796a6b13-4379-4833-85d0-51e55f122c51","namespace":"test-ns-9wtq2","name":"ray-cluster-small-group-worker-z7gbg"}
{"level":"info","ts":"2026-07-13T16:12:19.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"796a6b13-4379-4833-85d0-51e55f122c51","The worker Pod has already been deleted":"ray-cluster-small-group-worker-z7gbg"}
{"level":"info","ts":"2026-07-13T16:12:19.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"796a6b13-4379-4833-85d0-51e55f122c51","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:12:19.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"796a6b13-4379-4833-85d0-51e55f122c51","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:12:19.491Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"796a6b13-4379-4833-85d0-51e55f122c51","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:12:19.491Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"796a6b13-4379-4833-85d0-51e55f122c51","seconds":300}
{"level":"info","ts":"2026-07-13T16:12:20.990Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"958ffd36-85d4-41f0-8815-2e40ea7619cc"}
{"level":"info","ts":"2026-07-13T16:12:20.990Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"958ffd36-85d4-41f0-8815-2e40ea7619cc"}
{"level":"info","ts":"2026-07-13T16:12:20.990Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"958ffd36-85d4-41f0-8815-2e40ea7619cc","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:12:20.990Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"958ffd36-85d4-41f0-8815-2e40ea7619cc","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:12:20.990Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"958ffd36-85d4-41f0-8815-2e40ea7619cc","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:12:20.990Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"958ffd36-85d4-41f0-8815-2e40ea7619cc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-13T16:12:20.990Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"958ffd36-85d4-41f0-8815-2e40ea7619cc","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:12:20.990Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"958ffd36-85d4-41f0-8815-2e40ea7619cc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:12:20.990Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"958ffd36-85d4-41f0-8815-2e40ea7619cc","namespace":"test-ns-9wtq2","name":"ray-cluster-small-group-worker-z7gbg"}
{"level":"info","ts":"2026-07-13T16:12:20.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"958ffd36-85d4-41f0-8815-2e40ea7619cc","The worker Pod has already been deleted":"ray-cluster-small-group-worker-z7gbg"}
{"level":"info","ts":"2026-07-13T16:12:20.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"958ffd36-85d4-41f0-8815-2e40ea7619cc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:12:20.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"958ffd36-85d4-41f0-8815-2e40ea7619cc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:12:20.994Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"958ffd36-85d4-41f0-8815-2e40ea7619cc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:12:20.994Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"958ffd36-85d4-41f0-8815-2e40ea7619cc","seconds":300}
{"level":"info","ts":"2026-07-13T16:13:06.373Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"55b077ae-d22a-409c-8aa0-f8a659b0c7c4","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:06.374Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"55b077ae-d22a-409c-8aa0-f8a659b0c7c4","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:13:06.374Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"55b077ae-d22a-409c-8aa0-f8a659b0c7c4","pod":"ray-cluster-small-group-worker-tqz5h","ip":"10.134.0.37"}
{"level":"info","ts":"2026-07-13T16:13:06.398Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"55b077ae-d22a-409c-8aa0-f8a659b0c7c4","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-9wtq2.svc","ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local"],"ipAddresses":["10.133.0.40","10.134.0.37","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:13:06.410Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"55b077ae-d22a-409c-8aa0-f8a659b0c7c4","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-9wtq2.svc","ray-cluster-worker-svc.test-ns-9wtq2.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-9wtq2.svc","*.ray-cluster-worker-svc.test-ns-9wtq2.svc.cluster.local","*-worker-*.test-ns-9wtq2.svc","*-worker-*.test-ns-9wtq2.svc.cluster.local"],"ipAddresses":["10.133.0.40","10.134.0.37","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:13:06.410Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"55b077ae-d22a-409c-8aa0-f8a659b0c7c4","rayCluster":"ray-cluster","podIPs":["10.133.0.40","10.134.0.37"]}
{"level":"info","ts":"2026-07-13T16:13:29.323Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:29.323Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-9wtq2"}
{"level":"info","ts":"2026-07-13T16:13:29.334Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"94280a49-24aa-4685-bb50-a05cf3f22623","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:29.334Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"7b66320f-027c-411a-b2bf-fee656b53178"}
{"level":"info","ts":"2026-07-13T16:13:29.334Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"dd763bcd-96b4-44f3-a4e0-66df89ff32ba","namespacedName":{"name":"ray-cluster","namespace":"test-ns-9wtq2"}}
{"level":"info","ts":"2026-07-13T16:13:29.334Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"109ac6b2-98e9-44e4-bbd4-045070a1eacb","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:29.334Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"dd763bcd-96b4-44f3-a4e0-66df89ff32ba","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:29.334Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"dd763bcd-96b4-44f3-a4e0-66df89ff32ba","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:29.334Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"7b66320f-027c-411a-b2bf-fee656b53178"}
{"level":"info","ts":"2026-07-13T16:13:29.334Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"94280a49-24aa-4685-bb50-a05cf3f22623","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:13:29.334Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"94280a49-24aa-4685-bb50-a05cf3f22623","pod":"ray-cluster-small-group-worker-tqz5h","ip":"10.134.0.37"}
{"level":"info","ts":"2026-07-13T16:13:29.334Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"7b66320f-027c-411a-b2bf-fee656b53178","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:29.334Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"94280a49-24aa-4685-bb50-a05cf3f22623","rayCluster":"ray-cluster","podIPs":["10.133.0.40","10.134.0.37"]}
{"level":"info","ts":"2026-07-13T16:13:29.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"7b66320f-027c-411a-b2bf-fee656b53178","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:13:29.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"7b66320f-027c-411a-b2bf-fee656b53178","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:13:29.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"7b66320f-027c-411a-b2bf-fee656b53178","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:13:29.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"7b66320f-027c-411a-b2bf-fee656b53178","worker Pod":"ray-cluster-small-group-worker-tqz5h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-tqz5h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:13:29.334Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"7b66320f-027c-411a-b2bf-fee656b53178","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:29.334Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"7b66320f-027c-411a-b2bf-fee656b53178","namespace":"test-ns-9wtq2","name":"ray-cluster-small-group-worker-tqz5h"}
{"level":"info","ts":"2026-07-13T16:13:29.342Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"7b66320f-027c-411a-b2bf-fee656b53178","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:13:29.342Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"7b66320f-027c-411a-b2bf-fee656b53178","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:29.343Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"7b66320f-027c-411a-b2bf-fee656b53178","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:13:29.343Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"7b66320f-027c-411a-b2bf-fee656b53178","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:12:18Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:13:29Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-07-13T16:13:29.345Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"dd763bcd-96b4-44f3-a4e0-66df89ff32ba","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:29.346Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a62e18ce-f2b3-465f-bcfc-6bcfc8f462c6","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:29.346Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a62e18ce-f2b3-465f-bcfc-6bcfc8f462c6","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:13:29.346Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a62e18ce-f2b3-465f-bcfc-6bcfc8f462c6","pod":"ray-cluster-small-group-worker-tqz5h","ip":"10.134.0.37"}
{"level":"info","ts":"2026-07-13T16:13:29.346Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a62e18ce-f2b3-465f-bcfc-6bcfc8f462c6","rayCluster":"ray-cluster","podIPs":["10.133.0.40","10.134.0.37"]}
{"level":"info","ts":"2026-07-13T16:13:29.348Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"7b66320f-027c-411a-b2bf-fee656b53178","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-9wtq2","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-13T16:13:29.348Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"7b66320f-027c-411a-b2bf-fee656b53178"}
{"level":"error","ts":"2026-07-13T16:13:29.348Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"7b66320f-027c-411a-b2bf-fee656b53178","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:29.349Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9dafba1c-15a7-433d-930f-4f8993bbd0d6"}
{"level":"info","ts":"2026-07-13T16:13:29.349Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9dafba1c-15a7-433d-930f-4f8993bbd0d6"}
{"level":"info","ts":"2026-07-13T16:13:29.349Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9dafba1c-15a7-433d-930f-4f8993bbd0d6","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:29.349Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9dafba1c-15a7-433d-930f-4f8993bbd0d6","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:13:29.349Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9dafba1c-15a7-433d-930f-4f8993bbd0d6","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:13:29.349Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9dafba1c-15a7-433d-930f-4f8993bbd0d6","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:13:29.349Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9dafba1c-15a7-433d-930f-4f8993bbd0d6","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:13:29.349Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"9dafba1c-15a7-433d-930f-4f8993bbd0d6","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:12:18Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:13:29Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-07-13T16:13:29.351Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"109ac6b2-98e9-44e4-bbd4-045070a1eacb","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:13:29.361Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ac31ebfd-4758-46f4-94b0-c5e0563d14a5"}
{"level":"info","ts":"2026-07-13T16:13:29.361Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ac31ebfd-4758-46f4-94b0-c5e0563d14a5"}
{"level":"info","ts":"2026-07-13T16:13:29.361Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ac31ebfd-4758-46f4-94b0-c5e0563d14a5","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:29.361Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ac31ebfd-4758-46f4-94b0-c5e0563d14a5","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:13:29.361Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ac31ebfd-4758-46f4-94b0-c5e0563d14a5","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:13:29.361Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ac31ebfd-4758-46f4-94b0-c5e0563d14a5","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:13:29.361Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ac31ebfd-4758-46f4-94b0-c5e0563d14a5","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:13:29.361Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ac31ebfd-4758-46f4-94b0-c5e0563d14a5","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:12:18Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:13:29Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-07-13T16:13:29.361Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"72bdf1e5-5afc-4411-a903-7606b9b5db47","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:29.361Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"72bdf1e5-5afc-4411-a903-7606b9b5db47","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:13:29.361Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"72bdf1e5-5afc-4411-a903-7606b9b5db47","pod":"ray-cluster-small-group-worker-tqz5h","ip":"10.134.0.37"}
{"level":"info","ts":"2026-07-13T16:13:29.361Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"72bdf1e5-5afc-4411-a903-7606b9b5db47","rayCluster":"ray-cluster","podIPs":["10.133.0.40","10.134.0.37"]}
{"level":"info","ts":"2026-07-13T16:13:29.363Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"109ac6b2-98e9-44e4-bbd4-045070a1eacb","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:13:29.363Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"109ac6b2-98e9-44e4-bbd4-045070a1eacb","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:29.363Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"156b53ec-9440-448a-b593-914b89ed9efb","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:29.367Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ac31ebfd-4758-46f4-94b0-c5e0563d14a5","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-9wtq2","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-13T16:13:29.367Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ac31ebfd-4758-46f4-94b0-c5e0563d14a5"}
{"level":"error","ts":"2026-07-13T16:13:29.367Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ac31ebfd-4758-46f4-94b0-c5e0563d14a5","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:29.367Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cec4daf-f1ef-4c62-a4b0-c58eed257219"}
{"level":"info","ts":"2026-07-13T16:13:29.367Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cec4daf-f1ef-4c62-a4b0-c58eed257219"}
{"level":"info","ts":"2026-07-13T16:13:29.367Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cec4daf-f1ef-4c62-a4b0-c58eed257219","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:29.367Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cec4daf-f1ef-4c62-a4b0-c58eed257219","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:13:29.367Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cec4daf-f1ef-4c62-a4b0-c58eed257219","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:13:29.367Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cec4daf-f1ef-4c62-a4b0-c58eed257219","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:13:29.368Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cec4daf-f1ef-4c62-a4b0-c58eed257219","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:13:29.368Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"5cec4daf-f1ef-4c62-a4b0-c58eed257219","seconds":300}
{"level":"info","ts":"2026-07-13T16:13:29.421Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"31af63c3-1d1c-4b1d-8bb0-3a24a2ab1aaa"}
{"level":"info","ts":"2026-07-13T16:13:29.421Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"31af63c3-1d1c-4b1d-8bb0-3a24a2ab1aaa"}
{"level":"info","ts":"2026-07-13T16:13:29.421Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"31af63c3-1d1c-4b1d-8bb0-3a24a2ab1aaa","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:29.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"31af63c3-1d1c-4b1d-8bb0-3a24a2ab1aaa","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:13:29.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"31af63c3-1d1c-4b1d-8bb0-3a24a2ab1aaa","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:13:29.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"31af63c3-1d1c-4b1d-8bb0-3a24a2ab1aaa","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:13:29.421Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"31af63c3-1d1c-4b1d-8bb0-3a24a2ab1aaa","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:13:29.421Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"31af63c3-1d1c-4b1d-8bb0-3a24a2ab1aaa","seconds":300}
{"level":"info","ts":"2026-07-13T16:13:29.425Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"156b53ec-9440-448a-b593-914b89ed9efb","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:13:29.433Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"156b53ec-9440-448a-b593-914b89ed9efb","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:13:29.433Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"156b53ec-9440-448a-b593-914b89ed9efb","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:29.809Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3a20a70a-d28e-4d26-b4f6-c543dae67503"}
{"level":"info","ts":"2026-07-13T16:13:29.809Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3a20a70a-d28e-4d26-b4f6-c543dae67503"}
{"level":"info","ts":"2026-07-13T16:13:29.809Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3a20a70a-d28e-4d26-b4f6-c543dae67503","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:29.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3a20a70a-d28e-4d26-b4f6-c543dae67503","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:13:29.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3a20a70a-d28e-4d26-b4f6-c543dae67503","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:13:29.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3a20a70a-d28e-4d26-b4f6-c543dae67503","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:13:29.810Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3a20a70a-d28e-4d26-b4f6-c543dae67503","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:13:29.810Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3a20a70a-d28e-4d26-b4f6-c543dae67503","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:13:29Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:13:29Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-07-13T16:13:29.821Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"6d499a30-6bfa-422a-88b0-cb9c7eaa2a09","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:29.821Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"2df2c45f-d241-471a-96af-51daeeb98d25","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:29.821Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"6d499a30-6bfa-422a-88b0-cb9c7eaa2a09","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:13:29.821Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"6d499a30-6bfa-422a-88b0-cb9c7eaa2a09","pod":"ray-cluster-small-group-worker-tqz5h","ip":"10.134.0.37"}
{"level":"info","ts":"2026-07-13T16:13:29.821Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"6d499a30-6bfa-422a-88b0-cb9c7eaa2a09","rayCluster":"ray-cluster","podIPs":["10.133.0.40","10.134.0.37"]}
{"level":"info","ts":"2026-07-13T16:13:29.830Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"2df2c45f-d241-471a-96af-51daeeb98d25","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:13:29.840Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"2df2c45f-d241-471a-96af-51daeeb98d25","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:13:29.840Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"2df2c45f-d241-471a-96af-51daeeb98d25","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.001Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.001Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-4j8fw"}
{"level":"info","ts":"2026-07-13T16:13:30.011Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"e5d8450e-8570-4214-921b-9a67c5b5ca6b","namespacedName":{"name":"ray-cluster","namespace":"test-ns-4j8fw"}}
{"level":"info","ts":"2026-07-13T16:13:30.011Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"f78874e6-c90c-4feb-82e6-8e7ed1bc8cb8","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.011Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"1d98f024-5a06-4d8e-8819-2900ebf0eaa6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.011Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"e5d8450e-8570-4214-921b-9a67c5b5ca6b","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.011Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"e5d8450e-8570-4214-921b-9a67c5b5ca6b","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.011Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"f78874e6-c90c-4feb-82e6-8e7ed1bc8cb8","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.011Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"e5d8450e-8570-4214-921b-9a67c5b5ca6b","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.015Z","logger":"controllers.RayCluster","msg":"Created service account for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bc450b70-43ae-4bb4-9df1-4d2de1a58e86","name":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.017Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.017Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-4j8fw"}
{"level":"info","ts":"2026-07-13T16:13:30.022Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"f78874e6-c90c-4feb-82e6-8e7ed1bc8cb8","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.022Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"f78874e6-c90c-4feb-82e6-8e7ed1bc8cb8","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.028Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"1d98f024-5a06-4d8e-8819-2900ebf0eaa6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.028Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"79edb9dd-e1aa-4f8a-bddc-2911c754a30e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.030Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"f78874e6-c90c-4feb-82e6-8e7ed1bc8cb8","rayCluster":"ray-cluster","secret":"ca-secret"}
{"level":"info","ts":"2026-07-13T16:13:30.031Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"f78874e6-c90c-4feb-82e6-8e7ed1bc8cb8","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.034Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"e5d8450e-8570-4214-921b-9a67c5b5ca6b","name":"ray-cluster-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:13:30.034Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding does not exist yet, will retry auth SA patch","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"e5d8450e-8570-4214-921b-9a67c5b5ca6b","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.041Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"f78874e6-c90c-4feb-82e6-8e7ed1bc8cb8","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.043Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"79edb9dd-e1aa-4f8a-bddc-2911c754a30e","name":"ray-cluster-head"}
{"level":"error","ts":"2026-07-13T16:13:30.044Z","logger":"authentication-controller","msg":"Failed to update AuthenticationReady condition","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"e5d8450e-8570-4214-921b-9a67c5b5ca6b","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleOIDCConfiguration\n\t/workspace/controllers/ray/authentication_controller.go:285\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.044Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding not ready yet, requeueing","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"e5d8450e-8570-4214-921b-9a67c5b5ca6b","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.044Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"d890bac0-367c-437f-899c-505e7cbf5d4c","namespacedName":{"name":"ray-cluster","namespace":"test-ns-4j8fw"}}
{"level":"info","ts":"2026-07-13T16:13:30.044Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"d890bac0-367c-437f-899c-505e7cbf5d4c","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.044Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"d890bac0-367c-437f-899c-505e7cbf5d4c","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.044Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding does not exist yet, will retry auth SA patch","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"d890bac0-367c-437f-899c-505e7cbf5d4c","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.053Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"f78874e6-c90c-4feb-82e6-8e7ed1bc8cb8","rayCluster":"ray-cluster","podIPs":[]}
{"level":"info","ts":"2026-07-13T16:13:30.056Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding not ready yet, requeueing","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"d890bac0-367c-437f-899c-505e7cbf5d4c","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.121Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"79edb9dd-e1aa-4f8a-bddc-2911c754a30e","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:13:30.121Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"79edb9dd-e1aa-4f8a-bddc-2911c754a30e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.121Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"f78874e6-c90c-4feb-82e6-8e7ed1bc8cb8","rayCluster":"ray-cluster","podIPs":[]}
{"level":"error","ts":"2026-07-13T16:13:30.121Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"f78874e6-c90c-4feb-82e6-8e7ed1bc8cb8","error":"Certificate.cert-manager.io \"ray-worker-cert-ray-cluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.121Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"6c407270-f3b0-40a9-ac8e-3d1632bd537a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.121Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"f78874e6-c90c-4feb-82e6-8e7ed1bc8cb8"}
{"level":"error","ts":"2026-07-13T16:13:30.121Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"f78874e6-c90c-4feb-82e6-8e7ed1bc8cb8","error":"Certificate.cert-manager.io \"ray-worker-cert-ray-cluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.121Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"b4e1cc62-f7b9-4749-bee8-fe034d6bc911","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.132Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"6c407270-f3b0-40a9-ac8e-3d1632bd537a","name":"ray-cluster-head"}
{"level":"error","ts":"2026-07-13T16:13:30.133Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"b4e1cc62-f7b9-4749-bee8-fe034d6bc911","error":"certificates.cert-manager.io \"ray-worker-cert-ray-cluster\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:13:30.133Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"b4e1cc62-f7b9-4749-bee8-fe034d6bc911","error":"certificates.cert-manager.io \"ray-worker-cert-ray-cluster\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.133Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"b4e1cc62-f7b9-4749-bee8-fe034d6bc911"}
{"level":"error","ts":"2026-07-13T16:13:30.133Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"b4e1cc62-f7b9-4749-bee8-fe034d6bc911","error":"certificates.cert-manager.io \"ray-worker-cert-ray-cluster\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.133Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"92f35a81-9436-4653-9461-6ef1871cfb29","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.133Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"92f35a81-9436-4653-9461-6ef1871cfb29"}
{"level":"info","ts":"2026-07-13T16:13:30.143Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"50abefb9-258d-40d1-90dd-664284054bb1","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.143Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"50abefb9-258d-40d1-90dd-664284054bb1"}
{"level":"info","ts":"2026-07-13T16:13:30.144Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"6c407270-f3b0-40a9-ac8e-3d1632bd537a","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:13:30.144Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"6c407270-f3b0-40a9-ac8e-3d1632bd537a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.144Z","logger":"controllers.RayCluster","msg":"Created role for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bc450b70-43ae-4bb4-9df1-4d2de1a58e86","name":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.255Z","logger":"controllers.RayCluster","msg":"Created role binding for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bc450b70-43ae-4bb4-9df1-4d2de1a58e86","name":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.255Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bc450b70-43ae-4bb4-9df1-4d2de1a58e86"}
{"level":"info","ts":"2026-07-13T16:13:30.255Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bc450b70-43ae-4bb4-9df1-4d2de1a58e86"}
{"level":"info","ts":"2026-07-13T16:13:30.263Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bc450b70-43ae-4bb4-9df1-4d2de1a58e86","name":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:30.263Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bc450b70-43ae-4bb4-9df1-4d2de1a58e86","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-13T16:13:30.263Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bc450b70-43ae-4bb4-9df1-4d2de1a58e86","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.263Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bc450b70-43ae-4bb4-9df1-4d2de1a58e86","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-13T16:13:30.263Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bc450b70-43ae-4bb4-9df1-4d2de1a58e86","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.263Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bc450b70-43ae-4bb4-9df1-4d2de1a58e86","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.263Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"d2c35d7c-4c00-4ac0-9a52-68febd9508e0","namespacedName":{"name":"ray-cluster","namespace":"test-ns-4j8fw"}}
{"level":"info","ts":"2026-07-13T16:13:30.263Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bc450b70-43ae-4bb4-9df1-4d2de1a58e86","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.263Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bc450b70-43ae-4bb4-9df1-4d2de1a58e86","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:13:30.263Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"d2c35d7c-4c00-4ac0-9a52-68febd9508e0","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.263Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"d2c35d7c-4c00-4ac0-9a52-68febd9508e0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.263Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bc450b70-43ae-4bb4-9df1-4d2de1a58e86","name":"ray-cluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:13:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:13:30.268Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bc450b70-43ae-4bb4-9df1-4d2de1a58e86","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4j8fw","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-13T16:13:30.268Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bc450b70-43ae-4bb4-9df1-4d2de1a58e86"}
{"level":"error","ts":"2026-07-13T16:13:30.268Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bc450b70-43ae-4bb4-9df1-4d2de1a58e86","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.268Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"4ddaf641-bd7c-4c69-83b0-5e04f991fba0"}
{"level":"info","ts":"2026-07-13T16:13:30.268Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"4ddaf641-bd7c-4c69-83b0-5e04f991fba0"}
{"level":"info","ts":"2026-07-13T16:13:30.268Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"4ddaf641-bd7c-4c69-83b0-5e04f991fba0","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:30.268Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"4ddaf641-bd7c-4c69-83b0-5e04f991fba0","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-13T16:13:30.268Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"4ddaf641-bd7c-4c69-83b0-5e04f991fba0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.268Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"4ddaf641-bd7c-4c69-83b0-5e04f991fba0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-13T16:13:30.268Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"4ddaf641-bd7c-4c69-83b0-5e04f991fba0","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.268Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"4ddaf641-bd7c-4c69-83b0-5e04f991fba0","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.268Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"4ddaf641-bd7c-4c69-83b0-5e04f991fba0","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.268Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"4ddaf641-bd7c-4c69-83b0-5e04f991fba0","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:13:30.268Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"4ddaf641-bd7c-4c69-83b0-5e04f991fba0","name":"ray-cluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"}]},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:13:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:13:30.279Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"4ddaf641-bd7c-4c69-83b0-5e04f991fba0"}
{"level":"error","ts":"2026-07-13T16:13:30.279Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"4ddaf641-bd7c-4c69-83b0-5e04f991fba0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.279Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cae82b28-a5b9-481f-a969-0460e0bfc6da"}
{"level":"info","ts":"2026-07-13T16:13:30.279Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cae82b28-a5b9-481f-a969-0460e0bfc6da"}
{"level":"info","ts":"2026-07-13T16:13:30.279Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cae82b28-a5b9-481f-a969-0460e0bfc6da","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:30.279Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cae82b28-a5b9-481f-a969-0460e0bfc6da","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-13T16:13:30.279Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cae82b28-a5b9-481f-a969-0460e0bfc6da","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.279Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cae82b28-a5b9-481f-a969-0460e0bfc6da","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-13T16:13:30.279Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"a8d4f5e8-a5e7-4d2a-92a4-e6b1fd02542f","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.279Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cae82b28-a5b9-481f-a969-0460e0bfc6da","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.279Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cae82b28-a5b9-481f-a969-0460e0bfc6da","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.279Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cae82b28-a5b9-481f-a969-0460e0bfc6da","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.279Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cae82b28-a5b9-481f-a969-0460e0bfc6da","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:13:30.279Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"4504cc57-60f7-4296-bcd3-5e761095e4d4","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.279Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"a8d4f5e8-a5e7-4d2a-92a4-e6b1fd02542f"}
{"level":"info","ts":"2026-07-13T16:13:30.279Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cae82b28-a5b9-481f-a969-0460e0bfc6da","name":"ray-cluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"}]},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:13:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:13:30.285Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cae82b28-a5b9-481f-a969-0460e0bfc6da","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4j8fw","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-13T16:13:30.285Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cae82b28-a5b9-481f-a969-0460e0bfc6da"}
{"level":"error","ts":"2026-07-13T16:13:30.285Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cae82b28-a5b9-481f-a969-0460e0bfc6da","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.288Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"4504cc57-60f7-4296-bcd3-5e761095e4d4","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:13:30.289Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d71855c0-2fa8-4fc9-b88a-0cdaf5b34ef0"}
{"level":"info","ts":"2026-07-13T16:13:30.289Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d71855c0-2fa8-4fc9-b88a-0cdaf5b34ef0"}
{"level":"info","ts":"2026-07-13T16:13:30.289Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d71855c0-2fa8-4fc9-b88a-0cdaf5b34ef0","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:30.289Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d71855c0-2fa8-4fc9-b88a-0cdaf5b34ef0","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-13T16:13:30.289Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d71855c0-2fa8-4fc9-b88a-0cdaf5b34ef0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.289Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d71855c0-2fa8-4fc9-b88a-0cdaf5b34ef0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-13T16:13:30.289Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d71855c0-2fa8-4fc9-b88a-0cdaf5b34ef0","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.289Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d71855c0-2fa8-4fc9-b88a-0cdaf5b34ef0","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.289Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d71855c0-2fa8-4fc9-b88a-0cdaf5b34ef0","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.289Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d71855c0-2fa8-4fc9-b88a-0cdaf5b34ef0"}
{"level":"error","ts":"2026-07-13T16:13:30.289Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d71855c0-2fa8-4fc9-b88a-0cdaf5b34ef0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.298Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"4504cc57-60f7-4296-bcd3-5e761095e4d4","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:13:30.298Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"4504cc57-60f7-4296-bcd3-5e761095e4d4","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.329Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ed3886ac-31df-44a2-8fc7-1c821d19505e"}
{"level":"info","ts":"2026-07-13T16:13:30.329Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ed3886ac-31df-44a2-8fc7-1c821d19505e"}
{"level":"info","ts":"2026-07-13T16:13:30.329Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ed3886ac-31df-44a2-8fc7-1c821d19505e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:30.329Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ed3886ac-31df-44a2-8fc7-1c821d19505e","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-13T16:13:30.329Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ed3886ac-31df-44a2-8fc7-1c821d19505e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.329Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ed3886ac-31df-44a2-8fc7-1c821d19505e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-13T16:13:30.329Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ed3886ac-31df-44a2-8fc7-1c821d19505e","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.329Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ed3886ac-31df-44a2-8fc7-1c821d19505e","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.329Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ed3886ac-31df-44a2-8fc7-1c821d19505e","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.329Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ed3886ac-31df-44a2-8fc7-1c821d19505e"}
{"level":"error","ts":"2026-07-13T16:13:30.329Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ed3886ac-31df-44a2-8fc7-1c821d19505e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.352Z","logger":"authentication-controller","msg":"Added auth proxy SA to autoscaler RoleBinding","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"d2c35d7c-4c00-4ac0-9a52-68febd9508e0","roleBinding":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"authServiceAccount":"ray-cluster-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:13:30.359Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"d2c35d7c-4c00-4ac0-9a52-68febd9508e0","name":"kuberay-gateway-access","namespace":"test-ns-4j8fw","operation":"created"}
{"level":"info","ts":"2026-07-13T16:13:30.366Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"d2c35d7c-4c00-4ac0-9a52-68febd9508e0","name":"test-ns-4j8fw-ray-cluster","operation":"created"}
{"level":"info","ts":"2026-07-13T16:13:30.372Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"d2c35d7c-4c00-4ac0-9a52-68febd9508e0","name":"kube-rbac-proxy-config-ray-cluster","operation":"created"}
{"level":"error","ts":"2026-07-13T16:13:30.377Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"d2c35d7c-4c00-4ac0-9a52-68febd9508e0","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:172\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.377Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"d2c35d7c-4c00-4ac0-9a52-68febd9508e0"}
{"level":"error","ts":"2026-07-13T16:13:30.377Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"d2c35d7c-4c00-4ac0-9a52-68febd9508e0","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.382Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"21d87690-2694-4ebc-b360-0d03b7d5b073","namespacedName":{"name":"ray-cluster","namespace":"test-ns-4j8fw"}}
{"level":"info","ts":"2026-07-13T16:13:30.382Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"21d87690-2694-4ebc-b360-0d03b7d5b073","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.382Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"21d87690-2694-4ebc-b360-0d03b7d5b073","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.394Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"21d87690-2694-4ebc-b360-0d03b7d5b073","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.394Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"9da006a3-8637-4fa7-822d-b673a0b11946","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.394Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"6824ecac-ef94-4843-a6b9-913197c0c52d","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.394Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"9da006a3-8637-4fa7-822d-b673a0b11946"}
{"level":"info","ts":"2026-07-13T16:13:30.403Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"6824ecac-ef94-4843-a6b9-913197c0c52d","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:13:30.410Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"325e89e1-4eff-41d8-a292-daf1cb61e4e9"}
{"level":"info","ts":"2026-07-13T16:13:30.410Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"325e89e1-4eff-41d8-a292-daf1cb61e4e9"}
{"level":"info","ts":"2026-07-13T16:13:30.410Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"325e89e1-4eff-41d8-a292-daf1cb61e4e9","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:30.410Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"325e89e1-4eff-41d8-a292-daf1cb61e4e9","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-13T16:13:30.410Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"325e89e1-4eff-41d8-a292-daf1cb61e4e9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.410Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"325e89e1-4eff-41d8-a292-daf1cb61e4e9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-13T16:13:30.410Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"325e89e1-4eff-41d8-a292-daf1cb61e4e9","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.410Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"325e89e1-4eff-41d8-a292-daf1cb61e4e9","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.410Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"325e89e1-4eff-41d8-a292-daf1cb61e4e9","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.410Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"325e89e1-4eff-41d8-a292-daf1cb61e4e9"}
{"level":"error","ts":"2026-07-13T16:13:30.410Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"325e89e1-4eff-41d8-a292-daf1cb61e4e9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.412Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"6824ecac-ef94-4843-a6b9-913197c0c52d","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:13:30.412Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"6824ecac-ef94-4843-a6b9-913197c0c52d","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:30.571Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cb290200-7c8f-48e0-9150-2377e4076ae3"}
{"level":"info","ts":"2026-07-13T16:13:30.571Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cb290200-7c8f-48e0-9150-2377e4076ae3"}
{"level":"info","ts":"2026-07-13T16:13:30.571Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cb290200-7c8f-48e0-9150-2377e4076ae3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:30.571Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cb290200-7c8f-48e0-9150-2377e4076ae3","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-13T16:13:30.571Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cb290200-7c8f-48e0-9150-2377e4076ae3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.571Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cb290200-7c8f-48e0-9150-2377e4076ae3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-13T16:13:30.571Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cb290200-7c8f-48e0-9150-2377e4076ae3","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.571Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cb290200-7c8f-48e0-9150-2377e4076ae3","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.571Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cb290200-7c8f-48e0-9150-2377e4076ae3","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.571Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cb290200-7c8f-48e0-9150-2377e4076ae3"}
{"level":"error","ts":"2026-07-13T16:13:30.571Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"cb290200-7c8f-48e0-9150-2377e4076ae3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.763Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a4fa9a40-d108-4b62-b752-615519590247"}
{"level":"info","ts":"2026-07-13T16:13:30.763Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a4fa9a40-d108-4b62-b752-615519590247"}
{"level":"info","ts":"2026-07-13T16:13:30.763Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a4fa9a40-d108-4b62-b752-615519590247","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:30.763Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a4fa9a40-d108-4b62-b752-615519590247","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:13:30.763Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a4fa9a40-d108-4b62-b752-615519590247","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:13:30.763Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a4fa9a40-d108-4b62-b752-615519590247","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:13:30.763Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a4fa9a40-d108-4b62-b752-615519590247","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:13:30.763Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a4fa9a40-d108-4b62-b752-615519590247","seconds":300}
{"level":"info","ts":"2026-07-13T16:13:30.768Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a07ae642-a020-446c-ab67-78548fe6e79d"}
{"level":"info","ts":"2026-07-13T16:13:30.768Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a07ae642-a020-446c-ab67-78548fe6e79d"}
{"level":"info","ts":"2026-07-13T16:13:30.768Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a07ae642-a020-446c-ab67-78548fe6e79d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:30.769Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a07ae642-a020-446c-ab67-78548fe6e79d","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:13:30.769Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a07ae642-a020-446c-ab67-78548fe6e79d","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:13:30.769Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a07ae642-a020-446c-ab67-78548fe6e79d","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:13:30.769Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a07ae642-a020-446c-ab67-78548fe6e79d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:30.769Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a07ae642-a020-446c-ab67-78548fe6e79d","namespace":"test-ns-9wtq2","name":"ray-cluster-small-group-worker-tqz5h"}
{"level":"info","ts":"2026-07-13T16:13:30.773Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a07ae642-a020-446c-ab67-78548fe6e79d","The worker Pod has already been deleted":"ray-cluster-small-group-worker-tqz5h"}
{"level":"info","ts":"2026-07-13T16:13:30.773Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a07ae642-a020-446c-ab67-78548fe6e79d","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:13:30.773Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a07ae642-a020-446c-ab67-78548fe6e79d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:30.773Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a07ae642-a020-446c-ab67-78548fe6e79d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:13:30.773Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"a07ae642-a020-446c-ab67-78548fe6e79d","seconds":300}
{"level":"info","ts":"2026-07-13T16:13:30.891Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e88f0af5-902e-4b14-928e-d327bff0505d"}
{"level":"info","ts":"2026-07-13T16:13:30.891Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e88f0af5-902e-4b14-928e-d327bff0505d"}
{"level":"info","ts":"2026-07-13T16:13:30.891Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e88f0af5-902e-4b14-928e-d327bff0505d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:30.891Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e88f0af5-902e-4b14-928e-d327bff0505d","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-13T16:13:30.891Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e88f0af5-902e-4b14-928e-d327bff0505d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:30.891Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e88f0af5-902e-4b14-928e-d327bff0505d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-13T16:13:30.891Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e88f0af5-902e-4b14-928e-d327bff0505d","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.891Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e88f0af5-902e-4b14-928e-d327bff0505d","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.891Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e88f0af5-902e-4b14-928e-d327bff0505d","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:30.891Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e88f0af5-902e-4b14-928e-d327bff0505d"}
{"level":"error","ts":"2026-07-13T16:13:30.891Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e88f0af5-902e-4b14-928e-d327bff0505d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:31.532Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a12a3491-4499-4f9c-afec-77455ff02154"}
{"level":"info","ts":"2026-07-13T16:13:31.532Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a12a3491-4499-4f9c-afec-77455ff02154"}
{"level":"info","ts":"2026-07-13T16:13:31.532Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a12a3491-4499-4f9c-afec-77455ff02154","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:31.532Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a12a3491-4499-4f9c-afec-77455ff02154","error":"head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"error","ts":"2026-07-13T16:13:31.532Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a12a3491-4499-4f9c-afec-77455ff02154","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:31.532Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a12a3491-4499-4f9c-afec-77455ff02154","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found"}
{"level":"info","ts":"2026-07-13T16:13:31.532Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a12a3491-4499-4f9c-afec-77455ff02154","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:31.533Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a12a3491-4499-4f9c-afec-77455ff02154","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:31.533Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a12a3491-4499-4f9c-afec-77455ff02154","filter labels":["test-ns-4j8fw",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:13:31.533Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a12a3491-4499-4f9c-afec-77455ff02154"}
{"level":"error","ts":"2026-07-13T16:13:31.533Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a12a3491-4499-4f9c-afec-77455ff02154","error":"mTLS secrets not ready: head certificate secret ray-head-secret-ray-cluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:13:31.821Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4e819bfb-b7f6-411b-a739-2989b6f95e06"}
{"level":"info","ts":"2026-07-13T16:13:31.821Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4e819bfb-b7f6-411b-a739-2989b6f95e06"}
{"level":"info","ts":"2026-07-13T16:13:31.821Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4e819bfb-b7f6-411b-a739-2989b6f95e06","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:31.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4e819bfb-b7f6-411b-a739-2989b6f95e06","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:13:31.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4e819bfb-b7f6-411b-a739-2989b6f95e06","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:13:31.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4e819bfb-b7f6-411b-a739-2989b6f95e06","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:13:31.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4e819bfb-b7f6-411b-a739-2989b6f95e06","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:31.821Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4e819bfb-b7f6-411b-a739-2989b6f95e06","namespace":"test-ns-9wtq2","name":"ray-cluster-small-group-worker-tqz5h"}
{"level":"info","ts":"2026-07-13T16:13:31.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4e819bfb-b7f6-411b-a739-2989b6f95e06","The worker Pod has already been deleted":"ray-cluster-small-group-worker-tqz5h"}
{"level":"info","ts":"2026-07-13T16:13:31.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4e819bfb-b7f6-411b-a739-2989b6f95e06","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:13:31.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4e819bfb-b7f6-411b-a739-2989b6f95e06","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:31.824Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4e819bfb-b7f6-411b-a739-2989b6f95e06","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:13:31.824Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4e819bfb-b7f6-411b-a739-2989b6f95e06","seconds":300}
{"level":"info","ts":"2026-07-13T16:13:32.814Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae"}
{"level":"info","ts":"2026-07-13T16:13:32.814Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae"}
{"level":"info","ts":"2026-07-13T16:13:32.814Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:32.814Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae"}
{"level":"info","ts":"2026-07-13T16:13:32.814Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:13:32.814Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae"}
{"level":"info","ts":"2026-07-13T16:13:32.814Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:32.814Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:13:32.814Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-13T16:13:32.814Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:13:32.814Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-4j8fw.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-13T16:13:32.814Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-4j8fw.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:13:32.814Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-4j8fw.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:13:32.814Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae","enabled":true}
{"level":"info","ts":"2026-07-13T16:13:32.834Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae","name":"ray-cluster-head-f82qs"}
{"level":"info","ts":"2026-07-13T16:13:32.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:13:32.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:32.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:13:32.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:32.834Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae","podName":"ray-cluster-head-f82qs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:13:32.834Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae","podName":"ray-cluster-head-f82qs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:13:32.834Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"ray-cluster-head-svc"},"newHead":{"podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"}}
{"level":"info","ts":"2026-07-13T16:13:32.834Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"159a8fb4-ca9b-4569-a5dc-d5e33ed65dae","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:13:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:13:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:13:32.845Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"c70dad45-f82b-414d-9a3a-610d8462b154","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:32.845Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"672e9c4f-b026-4f79-ab25-36eb6b8c69d9","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:32.845Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"27bd6ba4-ba21-47bb-8c58-48c8b21d0098"}
{"level":"info","ts":"2026-07-13T16:13:32.845Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"27bd6ba4-ba21-47bb-8c58-48c8b21d0098"}
{"level":"info","ts":"2026-07-13T16:13:32.845Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"c70dad45-f82b-414d-9a3a-610d8462b154","rayCluster":"ray-cluster","podIPs":[]}
{"level":"info","ts":"2026-07-13T16:13:32.845Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"27bd6ba4-ba21-47bb-8c58-48c8b21d0098","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:32.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"27bd6ba4-ba21-47bb-8c58-48c8b21d0098","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:13:32.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"27bd6ba4-ba21-47bb-8c58-48c8b21d0098","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:13:32.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"27bd6ba4-ba21-47bb-8c58-48c8b21d0098","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:13:32.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"27bd6ba4-ba21-47bb-8c58-48c8b21d0098","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:32.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"27bd6ba4-ba21-47bb-8c58-48c8b21d0098","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:13:32.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"27bd6ba4-ba21-47bb-8c58-48c8b21d0098","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:32.847Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"27bd6ba4-ba21-47bb-8c58-48c8b21d0098","podName":"ray-cluster-head-f82qs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:13:32.847Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"27bd6ba4-ba21-47bb-8c58-48c8b21d0098","podName":"ray-cluster-head-f82qs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:13:32.921Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"27bd6ba4-ba21-47bb-8c58-48c8b21d0098","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:13:32.921Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"27bd6ba4-ba21-47bb-8c58-48c8b21d0098","seconds":300}
{"level":"info","ts":"2026-07-13T16:13:32.923Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"66ece256-1a36-41d5-9489-67dff8db53a6"}
{"level":"info","ts":"2026-07-13T16:13:32.923Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"66ece256-1a36-41d5-9489-67dff8db53a6"}
{"level":"info","ts":"2026-07-13T16:13:32.923Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"66ece256-1a36-41d5-9489-67dff8db53a6","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:32.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"66ece256-1a36-41d5-9489-67dff8db53a6","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:13:32.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"66ece256-1a36-41d5-9489-67dff8db53a6","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:13:32.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"66ece256-1a36-41d5-9489-67dff8db53a6","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:13:32.926Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"672e9c4f-b026-4f79-ab25-36eb6b8c69d9","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:13:33.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"66ece256-1a36-41d5-9489-67dff8db53a6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:33.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"66ece256-1a36-41d5-9489-67dff8db53a6","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:13:33.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"66ece256-1a36-41d5-9489-67dff8db53a6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:33.021Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"66ece256-1a36-41d5-9489-67dff8db53a6","podName":"ray-cluster-head-f82qs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:13:33.021Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"66ece256-1a36-41d5-9489-67dff8db53a6","podName":"ray-cluster-head-f82qs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:13:33.022Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"66ece256-1a36-41d5-9489-67dff8db53a6","old conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"new conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}]}
{"level":"info","ts":"2026-07-13T16:13:33.022Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"66ece256-1a36-41d5-9489-67dff8db53a6","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:13:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:13:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:13:33.121Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"672e9c4f-b026-4f79-ab25-36eb6b8c69d9","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:13:33.121Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"672e9c4f-b026-4f79-ab25-36eb6b8c69d9","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:33.122Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"781d4823-4b9c-4efa-8b42-b097b42a4c7f","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:33.122Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"1ac2f00e-e8fe-49df-a811-b825905febd0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:33.122Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"781d4823-4b9c-4efa-8b42-b097b42a4c7f","rayCluster":"ray-cluster","podIPs":[]}
{"level":"info","ts":"2026-07-13T16:13:33.133Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"1ac2f00e-e8fe-49df-a811-b825905febd0","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:13:33.144Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"1ac2f00e-e8fe-49df-a811-b825905febd0","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:13:33.144Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"1ac2f00e-e8fe-49df-a811-b825905febd0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:33.288Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"adbf52ac-d924-4493-9452-5e5d62702ba7"}
{"level":"info","ts":"2026-07-13T16:13:33.288Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"adbf52ac-d924-4493-9452-5e5d62702ba7"}
{"level":"info","ts":"2026-07-13T16:13:33.288Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"adbf52ac-d924-4493-9452-5e5d62702ba7","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:33.288Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"adbf52ac-d924-4493-9452-5e5d62702ba7","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:13:33.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"adbf52ac-d924-4493-9452-5e5d62702ba7","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:13:33.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"adbf52ac-d924-4493-9452-5e5d62702ba7","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:13:33.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"adbf52ac-d924-4493-9452-5e5d62702ba7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:33.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"adbf52ac-d924-4493-9452-5e5d62702ba7","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:13:33.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"adbf52ac-d924-4493-9452-5e5d62702ba7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:33.289Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"adbf52ac-d924-4493-9452-5e5d62702ba7","podName":"ray-cluster-head-f82qs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:13:33.289Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"adbf52ac-d924-4493-9452-5e5d62702ba7","podName":"ray-cluster-head-f82qs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:13:33.289Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"adbf52ac-d924-4493-9452-5e5d62702ba7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:13:33.289Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"adbf52ac-d924-4493-9452-5e5d62702ba7","seconds":300}
{"level":"info","ts":"2026-07-13T16:13:33.785Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"002af5a0-118b-48ac-b521-a2b724917112"}
{"level":"info","ts":"2026-07-13T16:13:33.785Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"002af5a0-118b-48ac-b521-a2b724917112"}
{"level":"info","ts":"2026-07-13T16:13:33.785Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"002af5a0-118b-48ac-b521-a2b724917112","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:33.785Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"002af5a0-118b-48ac-b521-a2b724917112","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:13:33.785Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"002af5a0-118b-48ac-b521-a2b724917112","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:13:33.785Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"002af5a0-118b-48ac-b521-a2b724917112","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:13:33.785Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"002af5a0-118b-48ac-b521-a2b724917112","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:33.785Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"002af5a0-118b-48ac-b521-a2b724917112","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:13:33.785Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"002af5a0-118b-48ac-b521-a2b724917112","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:33.785Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"002af5a0-118b-48ac-b521-a2b724917112","podName":"ray-cluster-head-f82qs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:13:33.785Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"002af5a0-118b-48ac-b521-a2b724917112","podName":"ray-cluster-head-f82qs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:13:33.785Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"002af5a0-118b-48ac-b521-a2b724917112","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"newHead":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"}}
{"level":"info","ts":"2026-07-13T16:13:33.785Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"002af5a0-118b-48ac-b521-a2b724917112","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:13:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:13:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:13:33.798Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"08988daa-722e-4ef2-a90b-4afd5d3c9b36","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:33.798Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"0df687d0-ae4b-42ab-9972-1f4636ecde7c","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:33.798Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"08988daa-722e-4ef2-a90b-4afd5d3c9b36","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:13:33.811Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"0df687d0-ae4b-42ab-9972-1f4636ecde7c","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:13:33.812Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"08988daa-722e-4ef2-a90b-4afd5d3c9b36","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-4j8fw.svc","ray-cluster-head-svc.test-ns-4j8fw.svc.cluster.local"],"ipAddresses":["10.134.0.39","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:13:33.822Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"0df687d0-ae4b-42ab-9972-1f4636ecde7c","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:13:33.822Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"0df687d0-ae4b-42ab-9972-1f4636ecde7c","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:33.823Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"08988daa-722e-4ef2-a90b-4afd5d3c9b36","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-4j8fw.svc","ray-cluster-worker-svc.test-ns-4j8fw.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-4j8fw.svc","*.ray-cluster-worker-svc.test-ns-4j8fw.svc.cluster.local","*-worker-*.test-ns-4j8fw.svc","*-worker-*.test-ns-4j8fw.svc.cluster.local"],"ipAddresses":["10.134.0.39","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:13:33.823Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"08988daa-722e-4ef2-a90b-4afd5d3c9b36","rayCluster":"ray-cluster","podIPs":["10.134.0.39"]}
{"level":"info","ts":"2026-07-13T16:13:34.845Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"51d62401-c132-4233-864b-68555f10071b"}
{"level":"info","ts":"2026-07-13T16:13:34.845Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"51d62401-c132-4233-864b-68555f10071b"}
{"level":"info","ts":"2026-07-13T16:13:34.846Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"51d62401-c132-4233-864b-68555f10071b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:34.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"51d62401-c132-4233-864b-68555f10071b","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:13:34.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"51d62401-c132-4233-864b-68555f10071b","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:13:34.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"51d62401-c132-4233-864b-68555f10071b","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:13:34.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"51d62401-c132-4233-864b-68555f10071b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:34.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"51d62401-c132-4233-864b-68555f10071b","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:13:34.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"51d62401-c132-4233-864b-68555f10071b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:34.846Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"51d62401-c132-4233-864b-68555f10071b","podName":"ray-cluster-head-f82qs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:13:34.846Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"51d62401-c132-4233-864b-68555f10071b","podName":"ray-cluster-head-f82qs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:13:34.846Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"51d62401-c132-4233-864b-68555f10071b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:13:34.846Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"51d62401-c132-4233-864b-68555f10071b","seconds":300}
{"level":"info","ts":"2026-07-13T16:13:34.994Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"06ef1350-a9a1-495f-8dbe-97448ea11e03"}
{"level":"info","ts":"2026-07-13T16:13:34.994Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"06ef1350-a9a1-495f-8dbe-97448ea11e03"}
{"level":"info","ts":"2026-07-13T16:13:34.994Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"06ef1350-a9a1-495f-8dbe-97448ea11e03","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:34.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"06ef1350-a9a1-495f-8dbe-97448ea11e03","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:13:34.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"06ef1350-a9a1-495f-8dbe-97448ea11e03","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:13:34.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"06ef1350-a9a1-495f-8dbe-97448ea11e03","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:13:34.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"06ef1350-a9a1-495f-8dbe-97448ea11e03","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:34.994Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"06ef1350-a9a1-495f-8dbe-97448ea11e03","namespace":"test-ns-9wtq2","name":"ray-cluster-small-group-worker-tqz5h"}
{"level":"info","ts":"2026-07-13T16:13:35.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"06ef1350-a9a1-495f-8dbe-97448ea11e03","The worker Pod has already been deleted":"ray-cluster-small-group-worker-tqz5h"}
{"level":"info","ts":"2026-07-13T16:13:35.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"06ef1350-a9a1-495f-8dbe-97448ea11e03","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:13:35.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"06ef1350-a9a1-495f-8dbe-97448ea11e03","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:35.000Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"06ef1350-a9a1-495f-8dbe-97448ea11e03","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:13:35.000Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"06ef1350-a9a1-495f-8dbe-97448ea11e03","seconds":300}
{"level":"info","ts":"2026-07-13T16:13:35.020Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4c8d629c-3182-4ce2-822a-9addb22be693"}
{"level":"info","ts":"2026-07-13T16:13:35.020Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4c8d629c-3182-4ce2-822a-9addb22be693"}
{"level":"info","ts":"2026-07-13T16:13:35.020Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4c8d629c-3182-4ce2-822a-9addb22be693","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:35.020Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4c8d629c-3182-4ce2-822a-9addb22be693","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:13:35.020Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4c8d629c-3182-4ce2-822a-9addb22be693","head Pod":"ray-cluster-head-xmqjl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-xmqjl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:13:35.020Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4c8d629c-3182-4ce2-822a-9addb22be693","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:13:35.020Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4c8d629c-3182-4ce2-822a-9addb22be693","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:35.020Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4c8d629c-3182-4ce2-822a-9addb22be693","namespace":"test-ns-9wtq2","name":"ray-cluster-small-group-worker-tqz5h"}
{"level":"info","ts":"2026-07-13T16:13:35.023Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4c8d629c-3182-4ce2-822a-9addb22be693","The worker Pod has already been deleted":"ray-cluster-small-group-worker-tqz5h"}
{"level":"info","ts":"2026-07-13T16:13:35.023Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4c8d629c-3182-4ce2-822a-9addb22be693","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:13:35.023Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4c8d629c-3182-4ce2-822a-9addb22be693","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:35.023Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4c8d629c-3182-4ce2-822a-9addb22be693","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:13:35.023Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4c8d629c-3182-4ce2-822a-9addb22be693","seconds":300}
{"level":"info","ts":"2026-07-13T16:13:38.804Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1d0a95de-10be-4751-93c5-bceeec4f7de3"}
{"level":"info","ts":"2026-07-13T16:13:38.804Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1d0a95de-10be-4751-93c5-bceeec4f7de3"}
{"level":"info","ts":"2026-07-13T16:13:38.804Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1d0a95de-10be-4751-93c5-bceeec4f7de3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:38.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1d0a95de-10be-4751-93c5-bceeec4f7de3","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:13:38.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1d0a95de-10be-4751-93c5-bceeec4f7de3","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:13:38.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1d0a95de-10be-4751-93c5-bceeec4f7de3","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:13:38.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1d0a95de-10be-4751-93c5-bceeec4f7de3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:38.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1d0a95de-10be-4751-93c5-bceeec4f7de3","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:13:38.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1d0a95de-10be-4751-93c5-bceeec4f7de3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:38.804Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1d0a95de-10be-4751-93c5-bceeec4f7de3","podName":"ray-cluster-head-f82qs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:13:38.804Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1d0a95de-10be-4751-93c5-bceeec4f7de3","podName":"ray-cluster-head-f82qs","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:13:38.804Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1d0a95de-10be-4751-93c5-bceeec4f7de3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:13:38.804Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1d0a95de-10be-4751-93c5-bceeec4f7de3","seconds":300}
{"level":"info","ts":"2026-07-13T16:13:39.811Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"7eeb7fcb-ab04-43fe-9e09-e4714cc2a235"}
{"level":"info","ts":"2026-07-13T16:13:39.811Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"7eeb7fcb-ab04-43fe-9e09-e4714cc2a235"}
{"level":"info","ts":"2026-07-13T16:13:39.811Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"7eeb7fcb-ab04-43fe-9e09-e4714cc2a235","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:39.811Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"7eeb7fcb-ab04-43fe-9e09-e4714cc2a235","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:13:39.811Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"7eeb7fcb-ab04-43fe-9e09-e4714cc2a235","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:13:39.811Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"7eeb7fcb-ab04-43fe-9e09-e4714cc2a235","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:13:39.811Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"7eeb7fcb-ab04-43fe-9e09-e4714cc2a235","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:39.811Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"7eeb7fcb-ab04-43fe-9e09-e4714cc2a235","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:13:39.811Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"7eeb7fcb-ab04-43fe-9e09-e4714cc2a235","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:39.811Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"7eeb7fcb-ab04-43fe-9e09-e4714cc2a235","podName":"ray-cluster-head-f82qs","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:13:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:13:39.811Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"7eeb7fcb-ab04-43fe-9e09-e4714cc2a235","podName":"ray-cluster-head-f82qs","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:13:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:13:39.811Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"7eeb7fcb-ab04-43fe-9e09-e4714cc2a235","old conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"new conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}]}
{"level":"info","ts":"2026-07-13T16:13:39.811Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"7eeb7fcb-ab04-43fe-9e09-e4714cc2a235","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:13:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:13:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:13:39.822Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"05f2f8bb-3b92-4407-b3e6-73703b64546e","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:39.822Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"68ef7fa8-3bd9-4d1d-9cbb-e28fe0db1207","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:39.822Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"05f2f8bb-3b92-4407-b3e6-73703b64546e","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:13:39.822Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"05f2f8bb-3b92-4407-b3e6-73703b64546e","rayCluster":"ray-cluster","podIPs":["10.134.0.39"]}
{"level":"info","ts":"2026-07-13T16:13:39.836Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"68ef7fa8-3bd9-4d1d-9cbb-e28fe0db1207","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:13:39.845Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"68ef7fa8-3bd9-4d1d-9cbb-e28fe0db1207","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:13:39.845Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"68ef7fa8-3bd9-4d1d-9cbb-e28fe0db1207","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:13:41.822Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"dda1ad92-9061-43fd-b379-cb7d35ffd194"}
{"level":"info","ts":"2026-07-13T16:13:41.822Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"dda1ad92-9061-43fd-b379-cb7d35ffd194"}
{"level":"info","ts":"2026-07-13T16:13:41.822Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"dda1ad92-9061-43fd-b379-cb7d35ffd194","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:13:41.823Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"dda1ad92-9061-43fd-b379-cb7d35ffd194","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:13:41.823Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"dda1ad92-9061-43fd-b379-cb7d35ffd194","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:13:41.823Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"dda1ad92-9061-43fd-b379-cb7d35ffd194","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:13:41.823Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"dda1ad92-9061-43fd-b379-cb7d35ffd194","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:41.823Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"dda1ad92-9061-43fd-b379-cb7d35ffd194","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:13:41.823Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"dda1ad92-9061-43fd-b379-cb7d35ffd194","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:13:41.823Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"dda1ad92-9061-43fd-b379-cb7d35ffd194","podName":"ray-cluster-head-f82qs","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:13:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:13:41.823Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"dda1ad92-9061-43fd-b379-cb7d35ffd194","podName":"ray-cluster-head-f82qs","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:13:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-07-13T16:13:41.823Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"dda1ad92-9061-43fd-b379-cb7d35ffd194","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:13:41.823Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"dda1ad92-9061-43fd-b379-cb7d35ffd194","seconds":300}
{"level":"info","ts":"2026-07-13T16:14:00.144Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"651370ad-6635-4d73-b33a-351430bb72c9","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:00.144Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"651370ad-6635-4d73-b33a-351430bb72c9","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:14:00.144Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"651370ad-6635-4d73-b33a-351430bb72c9","rayCluster":"ray-cluster","podIPs":["10.134.0.39"]}
{"level":"info","ts":"2026-07-13T16:14:00.840Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"f70fa388-3bd6-449f-8005-001c6d80269b"}
{"level":"info","ts":"2026-07-13T16:14:00.840Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"f70fa388-3bd6-449f-8005-001c6d80269b"}
{"level":"info","ts":"2026-07-13T16:14:00.841Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"f70fa388-3bd6-449f-8005-001c6d80269b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:00.841Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"f70fa388-3bd6-449f-8005-001c6d80269b","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:14:00.841Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"f70fa388-3bd6-449f-8005-001c6d80269b","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:00.841Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"f70fa388-3bd6-449f-8005-001c6d80269b","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:14:00.841Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"f70fa388-3bd6-449f-8005-001c6d80269b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:00.841Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"f70fa388-3bd6-449f-8005-001c6d80269b","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:14:00.841Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"f70fa388-3bd6-449f-8005-001c6d80269b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:00.841Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"f70fa388-3bd6-449f-8005-001c6d80269b","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-07-13T16:14:00.841Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"f70fa388-3bd6-449f-8005-001c6d80269b","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:13:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:14:00Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-07-13T16:14:00.851Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"860637a4-51fb-445f-a8e2-70f2f8e5bf48","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:00.851Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5f20181a-1a16-4dab-b044-d3fa96d58fba","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:00.851Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"860637a4-51fb-445f-a8e2-70f2f8e5bf48","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:14:00.851Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"860637a4-51fb-445f-a8e2-70f2f8e5bf48","rayCluster":"ray-cluster","podIPs":["10.134.0.39"]}
{"level":"info","ts":"2026-07-13T16:14:00.860Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5f20181a-1a16-4dab-b044-d3fa96d58fba","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:14:00.870Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5f20181a-1a16-4dab-b044-d3fa96d58fba","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:14:00.870Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5f20181a-1a16-4dab-b044-d3fa96d58fba","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:02.852Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"31f011be-56fa-4b50-852b-70836c755570"}
{"level":"info","ts":"2026-07-13T16:14:02.852Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"31f011be-56fa-4b50-852b-70836c755570"}
{"level":"info","ts":"2026-07-13T16:14:02.852Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"31f011be-56fa-4b50-852b-70836c755570","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:02.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"31f011be-56fa-4b50-852b-70836c755570","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:14:02.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"31f011be-56fa-4b50-852b-70836c755570","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:02.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"31f011be-56fa-4b50-852b-70836c755570","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:14:02.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"31f011be-56fa-4b50-852b-70836c755570","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:02.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"31f011be-56fa-4b50-852b-70836c755570","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:14:02.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"31f011be-56fa-4b50-852b-70836c755570","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:02.853Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"31f011be-56fa-4b50-852b-70836c755570","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:14:02.853Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"31f011be-56fa-4b50-852b-70836c755570","seconds":300}
{"level":"info","ts":"2026-07-13T16:14:05.663Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9335b33-e63f-4c02-81fe-8b959c643238"}
{"level":"info","ts":"2026-07-13T16:14:05.663Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9335b33-e63f-4c02-81fe-8b959c643238"}
{"level":"info","ts":"2026-07-13T16:14:05.663Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9335b33-e63f-4c02-81fe-8b959c643238","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:05.663Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9335b33-e63f-4c02-81fe-8b959c643238","Found 1 head Pod":"ray-cluster-head-xmqjl","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-07-13 16:09:08 +0000 UTC,FinishedAt:2026-07-13 16:13:37 +0000 UTC,ContainerID:cri-o://cfe9c4d98668bf274fb91ea2bf3fef499febcd184462f20a9a27dd698b8ba057,}"}
{"level":"info","ts":"2026-07-13T16:14:05.663Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9335b33-e63f-4c02-81fe-8b959c643238","head Pod":"ray-cluster-head-xmqjl","shouldDelete":true,"reason":"The head Pod ray-cluster-head-xmqjl status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-07-13T16:14:05.669Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9335b33-e63f-4c02-81fe-8b959c643238","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod ray-cluster-head-xmqjl status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:05.669Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9335b33-e63f-4c02-81fe-8b959c643238","error":"The head Pod ray-cluster-head-xmqjl status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-07-13T16:14:05.669Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9335b33-e63f-4c02-81fe-8b959c643238","old conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"new conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:14:05Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}]}
{"level":"info","ts":"2026-07-13T16:14:05.669Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9335b33-e63f-4c02-81fe-8b959c643238","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:13:29Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:14:05Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:14:05Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-07-13T16:14:05.679Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9335b33-e63f-4c02-81fe-8b959c643238"}
{"level":"error","ts":"2026-07-13T16:14:05.679Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9335b33-e63f-4c02-81fe-8b959c643238","error":"The head Pod ray-cluster-head-xmqjl status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:05.680Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"715c4f0d-ab10-416a-b2c7-60017cdbdd14","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:05.680Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a1bbdbc6-8255-4331-9e4f-e28f47ca6642","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:05.680Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a1bbdbc6-8255-4331-9e4f-e28f47ca6642","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:14:05.684Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"67beb65b-3595-4d14-895d-8ce0af26d2a6"}
{"level":"info","ts":"2026-07-13T16:14:05.684Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"67beb65b-3595-4d14-895d-8ce0af26d2a6"}
{"level":"info","ts":"2026-07-13T16:14:05.684Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"67beb65b-3595-4d14-895d-8ce0af26d2a6","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:05.685Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"67beb65b-3595-4d14-895d-8ce0af26d2a6","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-13T16:14:05.685Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"67beb65b-3595-4d14-895d-8ce0af26d2a6","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:14:05.685Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"67beb65b-3595-4d14-895d-8ce0af26d2a6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:05.685Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"67beb65b-3595-4d14-895d-8ce0af26d2a6","namespace":"test-ns-9wtq2","name":"ray-cluster-small-group-worker-tqz5h"}
{"level":"info","ts":"2026-07-13T16:14:05.685Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"715c4f0d-ab10-416a-b2c7-60017cdbdd14","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:05.687Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"67beb65b-3595-4d14-895d-8ce0af26d2a6","The worker Pod has already been deleted":"ray-cluster-small-group-worker-tqz5h"}
{"level":"info","ts":"2026-07-13T16:14:05.687Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"67beb65b-3595-4d14-895d-8ce0af26d2a6","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:14:05.687Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"67beb65b-3595-4d14-895d-8ce0af26d2a6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:05.687Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"67beb65b-3595-4d14-895d-8ce0af26d2a6","podName":"ray-cluster-head-xmqjl","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-13T16:14:05.687Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"67beb65b-3595-4d14-895d-8ce0af26d2a6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:14:05.687Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"67beb65b-3595-4d14-895d-8ce0af26d2a6","seconds":300}
{"level":"info","ts":"2026-07-13T16:14:05.692Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a1bbdbc6-8255-4331-9e4f-e28f47ca6642","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-9wtq2.svc","ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local"],"ipAddresses":["10.133.0.40","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:14:05.703Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a1bbdbc6-8255-4331-9e4f-e28f47ca6642","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-9wtq2.svc","ray-cluster-worker-svc.test-ns-9wtq2.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-9wtq2.svc","*.ray-cluster-worker-svc.test-ns-9wtq2.svc.cluster.local","*-worker-*.test-ns-9wtq2.svc","*-worker-*.test-ns-9wtq2.svc.cluster.local"],"ipAddresses":["10.133.0.40","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:14:05.703Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a1bbdbc6-8255-4331-9e4f-e28f47ca6642","rayCluster":"ray-cluster","podIPs":["10.133.0.40"]}
{"level":"info","ts":"2026-07-13T16:14:06.411Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"3db3563c-fac4-48e2-8607-0c1e90b31c40","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:06.411Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"3db3563c-fac4-48e2-8607-0c1e90b31c40","pod":"ray-cluster-head-xmqjl","ip":"10.133.0.40"}
{"level":"info","ts":"2026-07-13T16:14:06.411Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"3db3563c-fac4-48e2-8607-0c1e90b31c40"}
{"level":"info","ts":"2026-07-13T16:14:06.438Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"47babc15-663a-494d-a980-194620d23c7e"}
{"level":"info","ts":"2026-07-13T16:14:06.438Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"47babc15-663a-494d-a980-194620d23c7e"}
{"level":"info","ts":"2026-07-13T16:14:06.439Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"47babc15-663a-494d-a980-194620d23c7e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:06.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"47babc15-663a-494d-a980-194620d23c7e","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-07-13T16:14:06.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"47babc15-663a-494d-a980-194620d23c7e","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:14:06.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"47babc15-663a-494d-a980-194620d23c7e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:06.439Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"47babc15-663a-494d-a980-194620d23c7e","namespace":"test-ns-9wtq2","name":"ray-cluster-small-group-worker-tqz5h"}
{"level":"info","ts":"2026-07-13T16:14:06.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"47babc15-663a-494d-a980-194620d23c7e","The worker Pod has already been deleted":"ray-cluster-small-group-worker-tqz5h"}
{"level":"info","ts":"2026-07-13T16:14:06.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"47babc15-663a-494d-a980-194620d23c7e","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:14:06.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"47babc15-663a-494d-a980-194620d23c7e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:06.442Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"47babc15-663a-494d-a980-194620d23c7e","podName":"ray-cluster-head-xmqjl","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-07-13T16:14:06.442Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"47babc15-663a-494d-a980-194620d23c7e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:14:06.442Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"47babc15-663a-494d-a980-194620d23c7e","seconds":300}
{"level":"info","ts":"2026-07-13T16:14:06.442Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051"}
{"level":"info","ts":"2026-07-13T16:14:06.442Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051"}
{"level":"info","ts":"2026-07-13T16:14:06.442Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:06.443Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051"}
{"level":"info","ts":"2026-07-13T16:14:06.443Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:06.443Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051"}
{"level":"info","ts":"2026-07-13T16:14:06.443Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:06.443Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:06.443Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-13T16:14:06.443Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:14:06.443Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-13T16:14:06.443Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:06.443Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:06.443Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051","enabled":true}
{"level":"error","ts":"2026-07-13T16:14:06.446Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:06.446Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:14:06.446Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:06.446Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:06.446Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:06.446Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"newHead":{"serviceName":"ray-cluster-head-svc"}}
{"level":"info","ts":"2026-07-13T16:14:06.446Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:14:05Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.40","serviceIP":"10.133.0.40","podName":"ray-cluster-head-xmqjl","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:14:05Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:14:06Z","stateTransitionTimes":{"ready":"2026-07-13T16:09:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-13T16:09:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-07-13T16:14:05Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:09:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:09:00Z","reason":"RayClusterSuspending","message":""},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-07-13T16:14:06Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated"}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-07-13T16:14:06.457Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051"}
{"level":"error","ts":"2026-07-13T16:14:06.457Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"97dc42e3-126c-4974-b818-91af6bb30051","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:06.457Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"547f8d43-1d0a-4794-8e34-3184efcfd906","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:06.457Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"af7f3b49-3ff8-4bfa-a1cc-3a13dd6b51f1","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:06.462Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"af7f3b49-3ff8-4bfa-a1cc-3a13dd6b51f1","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:06.463Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"bf5d7c2a-8fef-4ba0-a72c-530bb651ad7c"}
{"level":"info","ts":"2026-07-13T16:14:06.463Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"bf5d7c2a-8fef-4ba0-a72c-530bb651ad7c"}
{"level":"info","ts":"2026-07-13T16:14:06.463Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"bf5d7c2a-8fef-4ba0-a72c-530bb651ad7c","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:06.463Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"bf5d7c2a-8fef-4ba0-a72c-530bb651ad7c"}
{"level":"info","ts":"2026-07-13T16:14:06.463Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"bf5d7c2a-8fef-4ba0-a72c-530bb651ad7c","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:06.463Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"bf5d7c2a-8fef-4ba0-a72c-530bb651ad7c"}
{"level":"info","ts":"2026-07-13T16:14:06.463Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"bf5d7c2a-8fef-4ba0-a72c-530bb651ad7c","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:06.463Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"bf5d7c2a-8fef-4ba0-a72c-530bb651ad7c","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:06.463Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"bf5d7c2a-8fef-4ba0-a72c-530bb651ad7c","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-13T16:14:06.463Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"bf5d7c2a-8fef-4ba0-a72c-530bb651ad7c","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:14:06.463Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"bf5d7c2a-8fef-4ba0-a72c-530bb651ad7c","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-13T16:14:06.463Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"bf5d7c2a-8fef-4ba0-a72c-530bb651ad7c","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:06.463Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"bf5d7c2a-8fef-4ba0-a72c-530bb651ad7c","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:06.463Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"bf5d7c2a-8fef-4ba0-a72c-530bb651ad7c","enabled":true}
{"level":"error","ts":"2026-07-13T16:14:06.466Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"bf5d7c2a-8fef-4ba0-a72c-530bb651ad7c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:06.466Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"bf5d7c2a-8fef-4ba0-a72c-530bb651ad7c","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:14:06.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"bf5d7c2a-8fef-4ba0-a72c-530bb651ad7c","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:06.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"bf5d7c2a-8fef-4ba0-a72c-530bb651ad7c","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:06.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"bf5d7c2a-8fef-4ba0-a72c-530bb651ad7c","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:06.466Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"bf5d7c2a-8fef-4ba0-a72c-530bb651ad7c"}
{"level":"error","ts":"2026-07-13T16:14:06.466Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"bf5d7c2a-8fef-4ba0-a72c-530bb651ad7c","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:06.470Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"547f8d43-1d0a-4794-8e34-3184efcfd906","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-9wtq2.svc","ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:14:06.483Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"547f8d43-1d0a-4794-8e34-3184efcfd906","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-9wtq2.svc","ray-cluster-worker-svc.test-ns-9wtq2.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-9wtq2.svc","*.ray-cluster-worker-svc.test-ns-9wtq2.svc.cluster.local","*-worker-*.test-ns-9wtq2.svc","*-worker-*.test-ns-9wtq2.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:14:06.483Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"547f8d43-1d0a-4794-8e34-3184efcfd906"}
{"level":"info","ts":"2026-07-13T16:14:06.521Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"57d81d71-0d00-4adf-8f6f-215f0d13b08c"}
{"level":"info","ts":"2026-07-13T16:14:06.521Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"57d81d71-0d00-4adf-8f6f-215f0d13b08c"}
{"level":"info","ts":"2026-07-13T16:14:06.521Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"57d81d71-0d00-4adf-8f6f-215f0d13b08c","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:06.521Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"57d81d71-0d00-4adf-8f6f-215f0d13b08c"}
{"level":"info","ts":"2026-07-13T16:14:06.521Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"57d81d71-0d00-4adf-8f6f-215f0d13b08c","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:06.521Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"57d81d71-0d00-4adf-8f6f-215f0d13b08c"}
{"level":"info","ts":"2026-07-13T16:14:06.521Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"57d81d71-0d00-4adf-8f6f-215f0d13b08c","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:06.521Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"57d81d71-0d00-4adf-8f6f-215f0d13b08c","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:06.521Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"57d81d71-0d00-4adf-8f6f-215f0d13b08c","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-13T16:14:06.521Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"57d81d71-0d00-4adf-8f6f-215f0d13b08c","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:14:06.521Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"57d81d71-0d00-4adf-8f6f-215f0d13b08c","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-13T16:14:06.521Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"57d81d71-0d00-4adf-8f6f-215f0d13b08c","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:06.521Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"57d81d71-0d00-4adf-8f6f-215f0d13b08c","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:06.521Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"57d81d71-0d00-4adf-8f6f-215f0d13b08c","enabled":true}
{"level":"error","ts":"2026-07-13T16:14:06.524Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"57d81d71-0d00-4adf-8f6f-215f0d13b08c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:06.524Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"57d81d71-0d00-4adf-8f6f-215f0d13b08c","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:14:06.524Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"57d81d71-0d00-4adf-8f6f-215f0d13b08c","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:06.524Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"57d81d71-0d00-4adf-8f6f-215f0d13b08c","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:06.525Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"57d81d71-0d00-4adf-8f6f-215f0d13b08c","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:06.525Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"57d81d71-0d00-4adf-8f6f-215f0d13b08c"}
{"level":"error","ts":"2026-07-13T16:14:06.525Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"57d81d71-0d00-4adf-8f6f-215f0d13b08c","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:06.545Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9d9fba6-431f-40f1-9e1a-785a0144db90"}
{"level":"info","ts":"2026-07-13T16:14:06.545Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9d9fba6-431f-40f1-9e1a-785a0144db90"}
{"level":"info","ts":"2026-07-13T16:14:06.545Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9d9fba6-431f-40f1-9e1a-785a0144db90","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:06.545Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9d9fba6-431f-40f1-9e1a-785a0144db90"}
{"level":"info","ts":"2026-07-13T16:14:06.545Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9d9fba6-431f-40f1-9e1a-785a0144db90","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:06.545Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9d9fba6-431f-40f1-9e1a-785a0144db90"}
{"level":"info","ts":"2026-07-13T16:14:06.546Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9d9fba6-431f-40f1-9e1a-785a0144db90","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:06.546Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9d9fba6-431f-40f1-9e1a-785a0144db90","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:06.546Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9d9fba6-431f-40f1-9e1a-785a0144db90","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-13T16:14:06.546Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9d9fba6-431f-40f1-9e1a-785a0144db90","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:14:06.546Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9d9fba6-431f-40f1-9e1a-785a0144db90","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-13T16:14:06.546Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9d9fba6-431f-40f1-9e1a-785a0144db90","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:06.546Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9d9fba6-431f-40f1-9e1a-785a0144db90","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:06.546Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9d9fba6-431f-40f1-9e1a-785a0144db90","enabled":true}
{"level":"error","ts":"2026-07-13T16:14:06.549Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9d9fba6-431f-40f1-9e1a-785a0144db90","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:06.549Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9d9fba6-431f-40f1-9e1a-785a0144db90","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:14:06.549Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9d9fba6-431f-40f1-9e1a-785a0144db90","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:06.549Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9d9fba6-431f-40f1-9e1a-785a0144db90","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:06.549Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9d9fba6-431f-40f1-9e1a-785a0144db90","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:06.549Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9d9fba6-431f-40f1-9e1a-785a0144db90"}
{"level":"error","ts":"2026-07-13T16:14:06.549Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"c9d9fba6-431f-40f1-9e1a-785a0144db90","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:06.590Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"6267f249-47e9-4c5c-8d47-2612b87c9fd3"}
{"level":"info","ts":"2026-07-13T16:14:06.590Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"6267f249-47e9-4c5c-8d47-2612b87c9fd3"}
{"level":"info","ts":"2026-07-13T16:14:06.590Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"6267f249-47e9-4c5c-8d47-2612b87c9fd3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:06.590Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"6267f249-47e9-4c5c-8d47-2612b87c9fd3"}
{"level":"info","ts":"2026-07-13T16:14:06.590Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"6267f249-47e9-4c5c-8d47-2612b87c9fd3","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:06.590Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"6267f249-47e9-4c5c-8d47-2612b87c9fd3"}
{"level":"info","ts":"2026-07-13T16:14:06.590Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"6267f249-47e9-4c5c-8d47-2612b87c9fd3","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:06.590Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"6267f249-47e9-4c5c-8d47-2612b87c9fd3","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:06.590Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"6267f249-47e9-4c5c-8d47-2612b87c9fd3","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-13T16:14:06.590Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"6267f249-47e9-4c5c-8d47-2612b87c9fd3","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:14:06.590Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"6267f249-47e9-4c5c-8d47-2612b87c9fd3","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-13T16:14:06.590Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"6267f249-47e9-4c5c-8d47-2612b87c9fd3","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:06.590Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"6267f249-47e9-4c5c-8d47-2612b87c9fd3","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:06.590Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"6267f249-47e9-4c5c-8d47-2612b87c9fd3","enabled":true}
{"level":"error","ts":"2026-07-13T16:14:06.593Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"6267f249-47e9-4c5c-8d47-2612b87c9fd3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:06.593Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"6267f249-47e9-4c5c-8d47-2612b87c9fd3","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:14:06.593Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"6267f249-47e9-4c5c-8d47-2612b87c9fd3","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:06.593Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"6267f249-47e9-4c5c-8d47-2612b87c9fd3","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:06.593Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"6267f249-47e9-4c5c-8d47-2612b87c9fd3","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:06.593Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"6267f249-47e9-4c5c-8d47-2612b87c9fd3"}
{"level":"error","ts":"2026-07-13T16:14:06.593Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"6267f249-47e9-4c5c-8d47-2612b87c9fd3","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:06.674Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3342ed23-c4e1-49d7-8b42-093cf5841c25"}
{"level":"info","ts":"2026-07-13T16:14:06.674Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3342ed23-c4e1-49d7-8b42-093cf5841c25"}
{"level":"info","ts":"2026-07-13T16:14:06.674Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3342ed23-c4e1-49d7-8b42-093cf5841c25","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:06.674Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3342ed23-c4e1-49d7-8b42-093cf5841c25"}
{"level":"info","ts":"2026-07-13T16:14:06.674Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3342ed23-c4e1-49d7-8b42-093cf5841c25","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:06.674Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3342ed23-c4e1-49d7-8b42-093cf5841c25"}
{"level":"info","ts":"2026-07-13T16:14:06.674Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3342ed23-c4e1-49d7-8b42-093cf5841c25","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:06.674Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3342ed23-c4e1-49d7-8b42-093cf5841c25","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:06.674Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3342ed23-c4e1-49d7-8b42-093cf5841c25","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-13T16:14:06.674Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3342ed23-c4e1-49d7-8b42-093cf5841c25","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:14:06.674Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3342ed23-c4e1-49d7-8b42-093cf5841c25","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-13T16:14:06.674Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3342ed23-c4e1-49d7-8b42-093cf5841c25","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:06.674Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3342ed23-c4e1-49d7-8b42-093cf5841c25","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:06.674Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3342ed23-c4e1-49d7-8b42-093cf5841c25","enabled":true}
{"level":"error","ts":"2026-07-13T16:14:06.677Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3342ed23-c4e1-49d7-8b42-093cf5841c25","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:06.677Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3342ed23-c4e1-49d7-8b42-093cf5841c25","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:14:06.678Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3342ed23-c4e1-49d7-8b42-093cf5841c25","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:06.678Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3342ed23-c4e1-49d7-8b42-093cf5841c25","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:06.678Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3342ed23-c4e1-49d7-8b42-093cf5841c25","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:06.678Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3342ed23-c4e1-49d7-8b42-093cf5841c25"}
{"level":"error","ts":"2026-07-13T16:14:06.678Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3342ed23-c4e1-49d7-8b42-093cf5841c25","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:06.839Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ddd5e944-9cc1-4737-94e2-39b01256cd44"}
{"level":"info","ts":"2026-07-13T16:14:06.839Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ddd5e944-9cc1-4737-94e2-39b01256cd44"}
{"level":"info","ts":"2026-07-13T16:14:06.839Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ddd5e944-9cc1-4737-94e2-39b01256cd44","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:06.839Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ddd5e944-9cc1-4737-94e2-39b01256cd44"}
{"level":"info","ts":"2026-07-13T16:14:06.839Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ddd5e944-9cc1-4737-94e2-39b01256cd44","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:06.839Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ddd5e944-9cc1-4737-94e2-39b01256cd44"}
{"level":"info","ts":"2026-07-13T16:14:06.839Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ddd5e944-9cc1-4737-94e2-39b01256cd44","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:06.839Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ddd5e944-9cc1-4737-94e2-39b01256cd44","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:06.839Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ddd5e944-9cc1-4737-94e2-39b01256cd44","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-13T16:14:06.839Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ddd5e944-9cc1-4737-94e2-39b01256cd44","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:14:06.839Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ddd5e944-9cc1-4737-94e2-39b01256cd44","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-13T16:14:06.839Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ddd5e944-9cc1-4737-94e2-39b01256cd44","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:06.839Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ddd5e944-9cc1-4737-94e2-39b01256cd44","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:06.839Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ddd5e944-9cc1-4737-94e2-39b01256cd44","enabled":true}
{"level":"error","ts":"2026-07-13T16:14:06.842Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ddd5e944-9cc1-4737-94e2-39b01256cd44","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:06.842Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ddd5e944-9cc1-4737-94e2-39b01256cd44","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:14:06.842Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ddd5e944-9cc1-4737-94e2-39b01256cd44","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:06.842Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ddd5e944-9cc1-4737-94e2-39b01256cd44","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:06.842Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ddd5e944-9cc1-4737-94e2-39b01256cd44","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:06.842Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ddd5e944-9cc1-4737-94e2-39b01256cd44"}
{"level":"error","ts":"2026-07-13T16:14:06.842Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"ddd5e944-9cc1-4737-94e2-39b01256cd44","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:07.145Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:07.145Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-4j8fw"}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"ab087b3d-9fa9-4206-bd04-61f29e1e2902","namespacedName":{"name":"ray-cluster","namespace":"test-ns-4j8fw"}}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"375baa6a-0178-48b2-9784-8c0fe47ba7d2","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b"}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b"}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"ab087b3d-9fa9-4206-bd04-61f29e1e2902","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"ab087b3d-9fa9-4206-bd04-61f29e1e2902","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"cb99e1d6-884e-42e7-8c41-1eade411882a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"375baa6a-0178-48b2-9784-8c0fe47ba7d2","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b"}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"375baa6a-0178-48b2-9784-8c0fe47ba7d2","rayCluster":"ray-cluster","podIPs":["10.134.0.39"]}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b","nodeType":"worker","rayStartParams":{"address":"ray-cluster-head-svc.test-ns-4j8fw.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"1G"},"requests":{"cpu":"1","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b","rayStartCmd":"ray start  --address=ray-cluster-head-svc.test-ns-4j8fw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=ray-cluster-head-svc.test-ns-4j8fw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:14:07.155Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b","enabled":true}
{"level":"info","ts":"2026-07-13T16:14:07.167Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"ab087b3d-9fa9-4206-bd04-61f29e1e2902","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:07.168Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"8222ccf1-5c61-4c95-9c55-1c3fe72cfe42","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:07.168Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"8222ccf1-5c61-4c95-9c55-1c3fe72cfe42","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:14:07.168Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"8222ccf1-5c61-4c95-9c55-1c3fe72cfe42","rayCluster":"ray-cluster","podIPs":["10.134.0.39"]}
{"level":"info","ts":"2026-07-13T16:14:07.170Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"cb99e1d6-884e-42e7-8c41-1eade411882a","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:14:07.173Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b","name":"ray-cluster-small-group-worker-2hvgl"}
{"level":"info","ts":"2026-07-13T16:14:07.174Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b","podName":"ray-cluster-small-group-worker-2hvgl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:14:07.174Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:14:07.174Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:14:00Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:14:07Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-07-13T16:14:07.180Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"cb99e1d6-884e-42e7-8c41-1eade411882a","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:14:07.180Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"cb99e1d6-884e-42e7-8c41-1eade411882a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:07.180Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"9711c43e-df6a-49e6-9e23-ea34cded6a33","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:07.181Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4j8fw","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-13T16:14:07.181Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b"}
{"level":"error","ts":"2026-07-13T16:14:07.181Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"093306bb-ee04-4d62-8344-cbdd9d7ec88b","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:07.181Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a931d44-b956-43e0-a9e6-9e7a436487c9"}
{"level":"info","ts":"2026-07-13T16:14:07.181Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a931d44-b956-43e0-a9e6-9e7a436487c9"}
{"level":"info","ts":"2026-07-13T16:14:07.181Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a931d44-b956-43e0-a9e6-9e7a436487c9","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:07.181Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a931d44-b956-43e0-a9e6-9e7a436487c9"}
{"level":"info","ts":"2026-07-13T16:14:07.181Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a931d44-b956-43e0-a9e6-9e7a436487c9","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:07.181Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a931d44-b956-43e0-a9e6-9e7a436487c9"}
{"level":"info","ts":"2026-07-13T16:14:07.181Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a931d44-b956-43e0-a9e6-9e7a436487c9","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:07.181Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a931d44-b956-43e0-a9e6-9e7a436487c9","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:07.181Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a931d44-b956-43e0-a9e6-9e7a436487c9","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-13T16:14:07.181Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a931d44-b956-43e0-a9e6-9e7a436487c9","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:14:07.181Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a931d44-b956-43e0-a9e6-9e7a436487c9","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-13T16:14:07.181Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a931d44-b956-43e0-a9e6-9e7a436487c9","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:07.181Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a931d44-b956-43e0-a9e6-9e7a436487c9","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:07.181Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a931d44-b956-43e0-a9e6-9e7a436487c9","enabled":true}
{"level":"error","ts":"2026-07-13T16:14:07.184Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a931d44-b956-43e0-a9e6-9e7a436487c9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:07.185Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a931d44-b956-43e0-a9e6-9e7a436487c9","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:14:07.185Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a931d44-b956-43e0-a9e6-9e7a436487c9","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:07.185Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a931d44-b956-43e0-a9e6-9e7a436487c9","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:07.185Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a931d44-b956-43e0-a9e6-9e7a436487c9","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:07.185Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a931d44-b956-43e0-a9e6-9e7a436487c9"}
{"level":"error","ts":"2026-07-13T16:14:07.185Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"4a931d44-b956-43e0-a9e6-9e7a436487c9","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:07.185Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d9443183-bd4b-4378-94ca-bcbceffef270"}
{"level":"info","ts":"2026-07-13T16:14:07.185Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d9443183-bd4b-4378-94ca-bcbceffef270"}
{"level":"info","ts":"2026-07-13T16:14:07.185Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d9443183-bd4b-4378-94ca-bcbceffef270","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:07.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d9443183-bd4b-4378-94ca-bcbceffef270","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:14:07.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d9443183-bd4b-4378-94ca-bcbceffef270","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:07.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d9443183-bd4b-4378-94ca-bcbceffef270","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-13T16:14:07.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d9443183-bd4b-4378-94ca-bcbceffef270","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:07.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d9443183-bd4b-4378-94ca-bcbceffef270","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:07.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d9443183-bd4b-4378-94ca-bcbceffef270","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:14:07.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d9443183-bd4b-4378-94ca-bcbceffef270","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:07.221Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d9443183-bd4b-4378-94ca-bcbceffef270","podName":"ray-cluster-small-group-worker-2hvgl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:14:07.221Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d9443183-bd4b-4378-94ca-bcbceffef270","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:14:07.221Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d9443183-bd4b-4378-94ca-bcbceffef270","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:14:00Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:14:07Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-07-13T16:14:07.225Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"9711c43e-df6a-49e6-9e23-ea34cded6a33","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:14:07.233Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c8bebc5c-2908-4337-83df-453fad34be07"}
{"level":"info","ts":"2026-07-13T16:14:07.233Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c8bebc5c-2908-4337-83df-453fad34be07"}
{"level":"info","ts":"2026-07-13T16:14:07.233Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c8bebc5c-2908-4337-83df-453fad34be07","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:07.233Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c8bebc5c-2908-4337-83df-453fad34be07","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:14:07.233Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c8bebc5c-2908-4337-83df-453fad34be07","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:07.233Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c8bebc5c-2908-4337-83df-453fad34be07","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-13T16:14:07.233Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c8bebc5c-2908-4337-83df-453fad34be07","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:07.233Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c8bebc5c-2908-4337-83df-453fad34be07","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:07.233Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c8bebc5c-2908-4337-83df-453fad34be07","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:14:07.233Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c8bebc5c-2908-4337-83df-453fad34be07","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:07.233Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c8bebc5c-2908-4337-83df-453fad34be07","podName":"ray-cluster-small-group-worker-2hvgl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:14:07.234Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"2e6cb381-a36a-4d71-acb6-dce06293a6c3","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:07.321Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"2e6cb381-a36a-4d71-acb6-dce06293a6c3","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:14:07.321Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"2e6cb381-a36a-4d71-acb6-dce06293a6c3","rayCluster":"ray-cluster","podIPs":["10.134.0.39"]}
{"level":"info","ts":"2026-07-13T16:14:07.321Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c8bebc5c-2908-4337-83df-453fad34be07","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:14:07.321Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c8bebc5c-2908-4337-83df-453fad34be07","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:14:00Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:14:07Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-07-13T16:14:07.326Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"9711c43e-df6a-49e6-9e23-ea34cded6a33","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:14:07.326Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"9711c43e-df6a-49e6-9e23-ea34cded6a33","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:07.326Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"b150cdd4-707f-4e20-9e65-8b4febd101a7","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:07.421Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c8bebc5c-2908-4337-83df-453fad34be07","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4j8fw","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-13T16:14:07.421Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c8bebc5c-2908-4337-83df-453fad34be07"}
{"level":"error","ts":"2026-07-13T16:14:07.421Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c8bebc5c-2908-4337-83df-453fad34be07","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:07.426Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"77db6603-2d84-4031-871d-f9ddc2cb7e85"}
{"level":"info","ts":"2026-07-13T16:14:07.426Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"77db6603-2d84-4031-871d-f9ddc2cb7e85"}
{"level":"info","ts":"2026-07-13T16:14:07.426Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"b150cdd4-707f-4e20-9e65-8b4febd101a7","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:14:07.426Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"77db6603-2d84-4031-871d-f9ddc2cb7e85","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:07.427Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"77db6603-2d84-4031-871d-f9ddc2cb7e85","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:14:07.427Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"77db6603-2d84-4031-871d-f9ddc2cb7e85","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:07.427Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"77db6603-2d84-4031-871d-f9ddc2cb7e85","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-13T16:14:07.427Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"77db6603-2d84-4031-871d-f9ddc2cb7e85","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:07.427Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"77db6603-2d84-4031-871d-f9ddc2cb7e85","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:07.427Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"77db6603-2d84-4031-871d-f9ddc2cb7e85","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:14:07.427Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"77db6603-2d84-4031-871d-f9ddc2cb7e85","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:07.427Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"77db6603-2d84-4031-871d-f9ddc2cb7e85","podName":"ray-cluster-small-group-worker-2hvgl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:14:07.427Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"77db6603-2d84-4031-871d-f9ddc2cb7e85","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:14:07.427Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"77db6603-2d84-4031-871d-f9ddc2cb7e85","seconds":300}
{"level":"info","ts":"2026-07-13T16:14:07.437Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"b150cdd4-707f-4e20-9e65-8b4febd101a7","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:14:07.437Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"b150cdd4-707f-4e20-9e65-8b4febd101a7","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:07.618Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"34503a51-c2d8-4bc4-b2d6-61a248c25499"}
{"level":"info","ts":"2026-07-13T16:14:07.618Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"34503a51-c2d8-4bc4-b2d6-61a248c25499"}
{"level":"info","ts":"2026-07-13T16:14:07.618Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"34503a51-c2d8-4bc4-b2d6-61a248c25499","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:07.618Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"34503a51-c2d8-4bc4-b2d6-61a248c25499","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:14:07.618Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"34503a51-c2d8-4bc4-b2d6-61a248c25499","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:07.618Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"34503a51-c2d8-4bc4-b2d6-61a248c25499","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-13T16:14:07.618Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"34503a51-c2d8-4bc4-b2d6-61a248c25499","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:07.618Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"34503a51-c2d8-4bc4-b2d6-61a248c25499","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:07.618Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"34503a51-c2d8-4bc4-b2d6-61a248c25499","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:14:07.618Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"34503a51-c2d8-4bc4-b2d6-61a248c25499","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:07.618Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"34503a51-c2d8-4bc4-b2d6-61a248c25499","podName":"ray-cluster-small-group-worker-2hvgl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:14:07.618Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"34503a51-c2d8-4bc4-b2d6-61a248c25499","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:14:07.618Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"34503a51-c2d8-4bc4-b2d6-61a248c25499","seconds":300}
{"level":"info","ts":"2026-07-13T16:14:07.826Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3c890008-b4fa-4589-949b-a6f1365f5659"}
{"level":"info","ts":"2026-07-13T16:14:07.826Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3c890008-b4fa-4589-949b-a6f1365f5659"}
{"level":"info","ts":"2026-07-13T16:14:07.826Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3c890008-b4fa-4589-949b-a6f1365f5659","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:07.826Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3c890008-b4fa-4589-949b-a6f1365f5659"}
{"level":"info","ts":"2026-07-13T16:14:07.826Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3c890008-b4fa-4589-949b-a6f1365f5659","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:07.826Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3c890008-b4fa-4589-949b-a6f1365f5659"}
{"level":"info","ts":"2026-07-13T16:14:07.826Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3c890008-b4fa-4589-949b-a6f1365f5659","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:07.826Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3c890008-b4fa-4589-949b-a6f1365f5659","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:07.826Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3c890008-b4fa-4589-949b-a6f1365f5659","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-13T16:14:07.826Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3c890008-b4fa-4589-949b-a6f1365f5659","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:14:07.826Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3c890008-b4fa-4589-949b-a6f1365f5659","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-13T16:14:07.826Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3c890008-b4fa-4589-949b-a6f1365f5659","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:07.826Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3c890008-b4fa-4589-949b-a6f1365f5659","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:07.826Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3c890008-b4fa-4589-949b-a6f1365f5659","enabled":true}
{"level":"error","ts":"2026-07-13T16:14:07.830Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3c890008-b4fa-4589-949b-a6f1365f5659","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:07.830Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3c890008-b4fa-4589-949b-a6f1365f5659","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:14:07.830Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3c890008-b4fa-4589-949b-a6f1365f5659","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:07.830Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3c890008-b4fa-4589-949b-a6f1365f5659","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:07.830Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3c890008-b4fa-4589-949b-a6f1365f5659","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:07.830Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3c890008-b4fa-4589-949b-a6f1365f5659"}
{"level":"error","ts":"2026-07-13T16:14:07.830Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"3c890008-b4fa-4589-949b-a6f1365f5659","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:08.449Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e7376cb7-b305-42e1-9ed0-bcff1871b153"}
{"level":"info","ts":"2026-07-13T16:14:08.449Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e7376cb7-b305-42e1-9ed0-bcff1871b153"}
{"level":"info","ts":"2026-07-13T16:14:08.449Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e7376cb7-b305-42e1-9ed0-bcff1871b153","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:08.449Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e7376cb7-b305-42e1-9ed0-bcff1871b153","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:14:08.449Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e7376cb7-b305-42e1-9ed0-bcff1871b153","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:08.449Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e7376cb7-b305-42e1-9ed0-bcff1871b153","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-13T16:14:08.449Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e7376cb7-b305-42e1-9ed0-bcff1871b153","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:08.449Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e7376cb7-b305-42e1-9ed0-bcff1871b153","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:08.449Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e7376cb7-b305-42e1-9ed0-bcff1871b153","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:14:08.449Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e7376cb7-b305-42e1-9ed0-bcff1871b153","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:08.449Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e7376cb7-b305-42e1-9ed0-bcff1871b153","podName":"ray-cluster-small-group-worker-2hvgl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:14:08.450Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e7376cb7-b305-42e1-9ed0-bcff1871b153","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:14:08.450Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e7376cb7-b305-42e1-9ed0-bcff1871b153","seconds":300}
{"level":"info","ts":"2026-07-13T16:14:09.111Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"16ee195e-b723-4cb6-b0b6-46d9f2014f34"}
{"level":"info","ts":"2026-07-13T16:14:09.111Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"16ee195e-b723-4cb6-b0b6-46d9f2014f34"}
{"level":"info","ts":"2026-07-13T16:14:09.111Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"16ee195e-b723-4cb6-b0b6-46d9f2014f34","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:09.111Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"16ee195e-b723-4cb6-b0b6-46d9f2014f34"}
{"level":"info","ts":"2026-07-13T16:14:09.111Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"16ee195e-b723-4cb6-b0b6-46d9f2014f34","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:09.111Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"16ee195e-b723-4cb6-b0b6-46d9f2014f34"}
{"level":"info","ts":"2026-07-13T16:14:09.111Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"16ee195e-b723-4cb6-b0b6-46d9f2014f34","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:09.111Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"16ee195e-b723-4cb6-b0b6-46d9f2014f34","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:09.111Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"16ee195e-b723-4cb6-b0b6-46d9f2014f34","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-13T16:14:09.111Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"16ee195e-b723-4cb6-b0b6-46d9f2014f34","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:14:09.111Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"16ee195e-b723-4cb6-b0b6-46d9f2014f34","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-13T16:14:09.111Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"16ee195e-b723-4cb6-b0b6-46d9f2014f34","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:09.111Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"16ee195e-b723-4cb6-b0b6-46d9f2014f34","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:09.111Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"16ee195e-b723-4cb6-b0b6-46d9f2014f34","enabled":true}
{"level":"error","ts":"2026-07-13T16:14:09.116Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"16ee195e-b723-4cb6-b0b6-46d9f2014f34","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:09.117Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"16ee195e-b723-4cb6-b0b6-46d9f2014f34","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:14:09.117Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"16ee195e-b723-4cb6-b0b6-46d9f2014f34","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:09.117Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"16ee195e-b723-4cb6-b0b6-46d9f2014f34","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:09.117Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"16ee195e-b723-4cb6-b0b6-46d9f2014f34","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:09.117Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"16ee195e-b723-4cb6-b0b6-46d9f2014f34"}
{"level":"error","ts":"2026-07-13T16:14:09.117Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"16ee195e-b723-4cb6-b0b6-46d9f2014f34","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:11.677Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2ef3a831-d6ec-4886-bfe1-8c312e52f3b1"}
{"level":"info","ts":"2026-07-13T16:14:11.677Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2ef3a831-d6ec-4886-bfe1-8c312e52f3b1"}
{"level":"info","ts":"2026-07-13T16:14:11.678Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2ef3a831-d6ec-4886-bfe1-8c312e52f3b1","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:11.678Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2ef3a831-d6ec-4886-bfe1-8c312e52f3b1"}
{"level":"info","ts":"2026-07-13T16:14:11.678Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2ef3a831-d6ec-4886-bfe1-8c312e52f3b1","reason":"AuthenticationResourcesCreated","observedGeneration":5,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:11.678Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2ef3a831-d6ec-4886-bfe1-8c312e52f3b1"}
{"level":"info","ts":"2026-07-13T16:14:11.678Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2ef3a831-d6ec-4886-bfe1-8c312e52f3b1","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:11.678Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2ef3a831-d6ec-4886-bfe1-8c312e52f3b1","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:11.678Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2ef3a831-d6ec-4886-bfe1-8c312e52f3b1","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-07-13T16:14:11.678Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2ef3a831-d6ec-4886-bfe1-8c312e52f3b1","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-07-13T16:14:11.678Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2ef3a831-d6ec-4886-bfe1-8c312e52f3b1","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","node-ip-address":"ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-07-13T16:14:11.678Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2ef3a831-d6ec-4886-bfe1-8c312e52f3b1","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:11.678Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2ef3a831-d6ec-4886-bfe1-8c312e52f3b1","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --node-ip-address=ray-cluster-head-svc.test-ns-9wtq2.svc.cluster.local  --num-cpus=0 "}
{"level":"info","ts":"2026-07-13T16:14:11.678Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2ef3a831-d6ec-4886-bfe1-8c312e52f3b1","enabled":true}
{"level":"error","ts":"2026-07-13T16:14:11.681Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2ef3a831-d6ec-4886-bfe1-8c312e52f3b1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:11.681Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2ef3a831-d6ec-4886-bfe1-8c312e52f3b1","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated"}
{"level":"info","ts":"2026-07-13T16:14:11.681Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2ef3a831-d6ec-4886-bfe1-8c312e52f3b1","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:11.681Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2ef3a831-d6ec-4886-bfe1-8c312e52f3b1","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:11.682Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2ef3a831-d6ec-4886-bfe1-8c312e52f3b1","filter labels":["test-ns-9wtq2",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-07-13T16:14:11.682Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2ef3a831-d6ec-4886-bfe1-8c312e52f3b1"}
{"level":"error","ts":"2026-07-13T16:14:11.682Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"2ef3a831-d6ec-4886-bfe1-8c312e52f3b1","error":"FailedCreateHeadPod\npods \"ray-cluster-head-\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:12.268Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.268Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-4j8fw"}
{"level":"info","ts":"2026-07-13T16:14:12.278Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"bbcb6645-b266-4f24-b58e-eab60d48d2b2","namespacedName":{"name":"ray-cluster","namespace":"test-ns-4j8fw"}}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5f39440e-ace9-4975-8a81-bc203fc091b5","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"bbcb6645-b266-4f24-b58e-eab60d48d2b2","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"bbcb6645-b266-4f24-b58e-eab60d48d2b2","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"358ce120-932b-4313-8bb8-aec8eade9c53","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0"}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0"}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5f39440e-ace9-4975-8a81-bc203fc091b5","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0","workerReplicas":2,"NumOfHosts":1,"runningPods":1,"diff":1}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5f39440e-ace9-4975-8a81-bc203fc091b5","pod":"ray-cluster-small-group-worker-2hvgl","ip":"10.133.0.41"}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0"}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0","nodeType":"worker","rayStartParams":{"address":"ray-cluster-head-svc.test-ns-4j8fw.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"1G"},"requests":{"cpu":"1","memory":"1G"}}}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0","rayStartCmd":"ray start  --address=ray-cluster-head-svc.test-ns-4j8fw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=ray-cluster-head-svc.test-ns-4j8fw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-07-13T16:14:12.279Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0","enabled":true}
{"level":"info","ts":"2026-07-13T16:14:12.290Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"bbcb6645-b266-4f24-b58e-eab60d48d2b2","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.293Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5f39440e-ace9-4975-8a81-bc203fc091b5","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-4j8fw.svc","ray-cluster-head-svc.test-ns-4j8fw.svc.cluster.local"],"ipAddresses":["10.134.0.39","10.133.0.41","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:14:12.295Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0","name":"ray-cluster-small-group-worker-cgx8s"}
{"level":"info","ts":"2026-07-13T16:14:12.295Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:14:12.295Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:14:07Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:14:12Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-13T16:14:12.299Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"358ce120-932b-4313-8bb8-aec8eade9c53","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:14:12.308Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5f39440e-ace9-4975-8a81-bc203fc091b5","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-4j8fw.svc","ray-cluster-worker-svc.test-ns-4j8fw.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-4j8fw.svc","*.ray-cluster-worker-svc.test-ns-4j8fw.svc.cluster.local","*-worker-*.test-ns-4j8fw.svc","*-worker-*.test-ns-4j8fw.svc.cluster.local"],"ipAddresses":["10.134.0.39","10.133.0.41","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:14:12.308Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5f39440e-ace9-4975-8a81-bc203fc091b5","rayCluster":"ray-cluster","podIPs":["10.134.0.39","10.133.0.41"]}
{"level":"info","ts":"2026-07-13T16:14:12.308Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"e2329128-58eb-493c-b8fa-a8fb622c92d9","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.308Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4j8fw","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-13T16:14:12.308Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0"}
{"level":"error","ts":"2026-07-13T16:14:12.308Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"109eb9f2-5239-4fbc-a868-b9a271b673c0","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:12.308Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"e2329128-58eb-493c-b8fa-a8fb622c92d9","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:14:12.308Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"e2329128-58eb-493c-b8fa-a8fb622c92d9","pod":"ray-cluster-small-group-worker-2hvgl","ip":"10.133.0.41"}
{"level":"info","ts":"2026-07-13T16:14:12.308Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e31453c1-94dd-42ff-a36a-46dca277c093"}
{"level":"info","ts":"2026-07-13T16:14:12.308Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e31453c1-94dd-42ff-a36a-46dca277c093"}
{"level":"info","ts":"2026-07-13T16:14:12.308Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e31453c1-94dd-42ff-a36a-46dca277c093","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:12.308Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e31453c1-94dd-42ff-a36a-46dca277c093","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:14:12.308Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e31453c1-94dd-42ff-a36a-46dca277c093","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:12.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e31453c1-94dd-42ff-a36a-46dca277c093","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:14:12.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e31453c1-94dd-42ff-a36a-46dca277c093","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:12.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e31453c1-94dd-42ff-a36a-46dca277c093","worker Pod":"ray-cluster-small-group-worker-cgx8s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-cgx8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:12.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e31453c1-94dd-42ff-a36a-46dca277c093","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:12.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e31453c1-94dd-42ff-a36a-46dca277c093","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:14:12.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e31453c1-94dd-42ff-a36a-46dca277c093","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:12.309Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e31453c1-94dd-42ff-a36a-46dca277c093","podName":"ray-cluster-small-group-worker-2hvgl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:14:12.321Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e31453c1-94dd-42ff-a36a-46dca277c093","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:14:12.321Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"e31453c1-94dd-42ff-a36a-46dca277c093","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:14:07Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:14:12Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"error","ts":"2026-07-13T16:14:12.321Z","msg":"Failed to update certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"e2329128-58eb-493c-b8fa-a8fb622c92d9","error":"Operation cannot be fulfilled on certificates.cert-manager.io \"ray-worker-cert-ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).updateCertificateWithPodIPs\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:465\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:212\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:14:12.321Z","msg":"Failed to update worker certificate with pod IPs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"e2329128-58eb-493c-b8fa-a8fb622c92d9","error":"Operation cannot be fulfilled on certificates.cert-manager.io \"ray-worker-cert-ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:213\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:12.321Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"e2329128-58eb-493c-b8fa-a8fb622c92d9"}
{"level":"error","ts":"2026-07-13T16:14:12.321Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"e2329128-58eb-493c-b8fa-a8fb622c92d9","error":"Operation cannot be fulfilled on certificates.cert-manager.io \"ray-worker-cert-ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:12.326Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"358ce120-932b-4313-8bb8-aec8eade9c53","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:14:12.326Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"358ce120-932b-4313-8bb8-aec8eade9c53","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.326Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5a9d20a3-1b54-4fe1-a4c7-ca40d9e6b94b","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.326Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"2482448e-9f75-4a83-961b-32f14e6e5b94","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.326Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"2482448e-9f75-4a83-961b-32f14e6e5b94","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:14:12.326Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"2482448e-9f75-4a83-961b-32f14e6e5b94","pod":"ray-cluster-small-group-worker-2hvgl","ip":"10.133.0.41"}
{"level":"info","ts":"2026-07-13T16:14:12.326Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"2482448e-9f75-4a83-961b-32f14e6e5b94"}
{"level":"info","ts":"2026-07-13T16:14:12.333Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"9f60456d-64f4-4898-93be-12b168b8b605"}
{"level":"info","ts":"2026-07-13T16:14:12.333Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"9f60456d-64f4-4898-93be-12b168b8b605"}
{"level":"info","ts":"2026-07-13T16:14:12.333Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"9f60456d-64f4-4898-93be-12b168b8b605","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:12.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"9f60456d-64f4-4898-93be-12b168b8b605","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:14:12.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"9f60456d-64f4-4898-93be-12b168b8b605","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:12.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"9f60456d-64f4-4898-93be-12b168b8b605","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:14:12.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"9f60456d-64f4-4898-93be-12b168b8b605","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:12.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"9f60456d-64f4-4898-93be-12b168b8b605","worker Pod":"ray-cluster-small-group-worker-cgx8s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-cgx8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:12.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"9f60456d-64f4-4898-93be-12b168b8b605","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:12.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"9f60456d-64f4-4898-93be-12b168b8b605","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:14:12.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"9f60456d-64f4-4898-93be-12b168b8b605","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:12.333Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"9f60456d-64f4-4898-93be-12b168b8b605","podName":"ray-cluster-small-group-worker-2hvgl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:14:12.334Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"9f60456d-64f4-4898-93be-12b168b8b605","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:14:12.334Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"9f60456d-64f4-4898-93be-12b168b8b605","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:14:07Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:14:12Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-13T16:14:12.421Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"a868bd14-b468-4b12-9bd7-dc43dc288b74","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.421Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"a868bd14-b468-4b12-9bd7-dc43dc288b74","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:14:12.421Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"a868bd14-b468-4b12-9bd7-dc43dc288b74","pod":"ray-cluster-small-group-worker-2hvgl","ip":"10.133.0.41"}
{"level":"info","ts":"2026-07-13T16:14:12.421Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"a868bd14-b468-4b12-9bd7-dc43dc288b74"}
{"level":"info","ts":"2026-07-13T16:14:12.422Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"8fd9f8c3-99cc-4d7a-b85c-9f28aab95dce","namespacedName":{"name":"ray-cluster","namespace":"test-ns-9wtq2"}}
{"level":"info","ts":"2026-07-13T16:14:12.422Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"bdc8ca23-9a86-453a-b75f-3727838e64ff","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.422Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"8fd9f8c3-99cc-4d7a-b85c-9f28aab95dce","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.422Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"bdc8ca23-9a86-453a-b75f-3727838e64ff","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.422Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"8fd9f8c3-99cc-4d7a-b85c-9f28aab95dce","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.422Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"8fd9f8c3-99cc-4d7a-b85c-9f28aab95dce","namespace":"test-ns-9wtq2","otherAuthClusters":0,"action":"will delete"}
{"level":"error","ts":"2026-07-13T16:14:12.424Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"bdc8ca23-9a86-453a-b75f-3727838e64ff","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:14:12.424Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"bdc8ca23-9a86-453a-b75f-3727838e64ff","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:12.424Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"bdc8ca23-9a86-453a-b75f-3727838e64ff"}
{"level":"error","ts":"2026-07-13T16:14:12.424Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"bdc8ca23-9a86-453a-b75f-3727838e64ff","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:12.426Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5a9d20a3-1b54-4fe1-a4c7-ca40d9e6b94b","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:14:12.427Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"8fd9f8c3-99cc-4d7a-b85c-9f28aab95dce","name":"kuberay-gateway-access","namespace":"test-ns-9wtq2"}
{"level":"info","ts":"2026-07-13T16:14:12.428Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"9f60456d-64f4-4898-93be-12b168b8b605","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4j8fw","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-13T16:14:12.428Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"9f60456d-64f4-4898-93be-12b168b8b605"}
{"level":"error","ts":"2026-07-13T16:14:12.428Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"9f60456d-64f4-4898-93be-12b168b8b605","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:12.428Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"reconcileID":"139ec38c-0c1a-4fd5-8b1a-0eb4c86f576f"}
{"level":"info","ts":"2026-07-13T16:14:12.428Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ef7a0324-b1fa-4b5d-b265-d54fb88711c1"}
{"level":"info","ts":"2026-07-13T16:14:12.428Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ef7a0324-b1fa-4b5d-b265-d54fb88711c1"}
{"level":"info","ts":"2026-07-13T16:14:12.428Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ef7a0324-b1fa-4b5d-b265-d54fb88711c1","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:12.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ef7a0324-b1fa-4b5d-b265-d54fb88711c1","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:14:12.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ef7a0324-b1fa-4b5d-b265-d54fb88711c1","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:12.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ef7a0324-b1fa-4b5d-b265-d54fb88711c1","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:14:12.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ef7a0324-b1fa-4b5d-b265-d54fb88711c1","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:12.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ef7a0324-b1fa-4b5d-b265-d54fb88711c1","worker Pod":"ray-cluster-small-group-worker-cgx8s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-cgx8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:12.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ef7a0324-b1fa-4b5d-b265-d54fb88711c1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:12.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ef7a0324-b1fa-4b5d-b265-d54fb88711c1","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:14:12.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ef7a0324-b1fa-4b5d-b265-d54fb88711c1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:12.428Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ef7a0324-b1fa-4b5d-b265-d54fb88711c1","podName":"ray-cluster-small-group-worker-2hvgl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:14:12.428Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ef7a0324-b1fa-4b5d-b265-d54fb88711c1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:14:12.428Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ef7a0324-b1fa-4b5d-b265-d54fb88711c1","seconds":300}
{"level":"info","ts":"2026-07-13T16:14:12.430Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"25459eb7-d16f-4f20-a58e-7ead393bf8b8","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.430Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"25459eb7-d16f-4f20-a58e-7ead393bf8b8","rayCluster":"ray-cluster"}
{"level":"error","ts":"2026-07-13T16:14:12.432Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"25459eb7-d16f-4f20-a58e-7ead393bf8b8","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:14:12.432Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"25459eb7-d16f-4f20-a58e-7ead393bf8b8","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:12.432Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"25459eb7-d16f-4f20-a58e-7ead393bf8b8"}
{"level":"error","ts":"2026-07-13T16:14:12.432Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"25459eb7-d16f-4f20-a58e-7ead393bf8b8","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:12.433Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"2502a86d-387a-4932-b7c4-03d3f1f8148d"}
{"level":"info","ts":"2026-07-13T16:14:12.433Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"2502a86d-387a-4932-b7c4-03d3f1f8148d"}
{"level":"info","ts":"2026-07-13T16:14:12.433Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"2502a86d-387a-4932-b7c4-03d3f1f8148d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:12.433Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"2502a86d-387a-4932-b7c4-03d3f1f8148d","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:14:12.433Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"2502a86d-387a-4932-b7c4-03d3f1f8148d","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:12.433Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"2502a86d-387a-4932-b7c4-03d3f1f8148d","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:14:12.433Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"2502a86d-387a-4932-b7c4-03d3f1f8148d","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:12.433Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"2502a86d-387a-4932-b7c4-03d3f1f8148d","worker Pod":"ray-cluster-small-group-worker-cgx8s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-cgx8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:12.433Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"2502a86d-387a-4932-b7c4-03d3f1f8148d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:12.433Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"2502a86d-387a-4932-b7c4-03d3f1f8148d","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:14:12.433Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"2502a86d-387a-4932-b7c4-03d3f1f8148d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:12.434Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"2502a86d-387a-4932-b7c4-03d3f1f8148d","podName":"ray-cluster-small-group-worker-2hvgl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:14:12.434Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"2502a86d-387a-4932-b7c4-03d3f1f8148d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:14:12.434Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"2502a86d-387a-4932-b7c4-03d3f1f8148d","seconds":300}
{"level":"info","ts":"2026-07-13T16:14:12.435Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"8fd9f8c3-99cc-4d7a-b85c-9f28aab95dce","name":"test-ns-9wtq2-ray-cluster","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-13T16:14:12.437Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5a9d20a3-1b54-4fe1-a4c7-ca40d9e6b94b","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:14:12.437Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5a9d20a3-1b54-4fe1-a4c7-ca40d9e6b94b","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.437Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"32d8df92-b5b5-4267-b08e-90d62bf4190c"}
{"level":"info","ts":"2026-07-13T16:14:12.437Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"f4490215-1d5c-454c-b3ce-91b157ce457c","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.443Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"31174e92-073a-492c-9726-c2489dba191b","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.443Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"31174e92-073a-492c-9726-c2489dba191b","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.521Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"8fd9f8c3-99cc-4d7a-b85c-9f28aab95dce","configMap":"kube-rbac-proxy-config-ray-cluster"}
{"level":"error","ts":"2026-07-13T16:14:12.524Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"31174e92-073a-492c-9726-c2489dba191b","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:14:12.524Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"31174e92-073a-492c-9726-c2489dba191b","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:12.524Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"31174e92-073a-492c-9726-c2489dba191b"}
{"level":"error","ts":"2026-07-13T16:14:12.524Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"31174e92-073a-492c-9726-c2489dba191b","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:12.532Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"8fd9f8c3-99cc-4d7a-b85c-9f28aab95dce","serviceAccount":"ray-cluster-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-07-13T16:14:12.537Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"f4490215-1d5c-454c-b3ce-91b157ce457c","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:14:12.539Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.539Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-9wtq2"}
{"level":"info","ts":"2026-07-13T16:14:12.544Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a25131be-7c47-421c-a933-073431a48945","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.544Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a25131be-7c47-421c-a933-073431a48945","rayCluster":"ray-cluster"}
{"level":"error","ts":"2026-07-13T16:14:12.547Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a25131be-7c47-421c-a933-073431a48945","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T16:14:12.547Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a25131be-7c47-421c-a933-073431a48945","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:12.547Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a25131be-7c47-421c-a933-073431a48945"}
{"level":"error","ts":"2026-07-13T16:14:12.547Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a25131be-7c47-421c-a933-073431a48945","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-9wtq2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:12.548Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"f4490215-1d5c-454c-b3ce-91b157ce457c","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:14:12.548Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"f4490215-1d5c-454c-b3ce-91b157ce457c","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.549Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"8fd9f8c3-99cc-4d7a-b85c-9f28aab95dce","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.549Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"023f64da-7b35-4990-b404-38d2cd46941a","namespacedName":{"name":"ray-cluster","namespace":"test-ns-9wtq2"}}
{"level":"info","ts":"2026-07-13T16:14:12.549Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"023f64da-7b35-4990-b404-38d2cd46941a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.549Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"023f64da-7b35-4990-b404-38d2cd46941a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.549Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"023f64da-7b35-4990-b404-38d2cd46941a","namespace":"test-ns-9wtq2","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-07-13T16:14:12.549Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"023f64da-7b35-4990-b404-38d2cd46941a","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-07-13T16:14:12.549Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"73e07099-56f1-4d2c-94be-191701f881a4"}
{"level":"info","ts":"2026-07-13T16:14:12.549Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"2fcd3bce-017d-4548-8284-d00a25e3af32","namespace":"test-ns-9wtq2","clusterName":"ray-cluster"}
{"level":"error","ts":"2026-07-13T16:14:12.553Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"023f64da-7b35-4990-b404-38d2cd46941a","error":"rayclusters.ray.io \"ray-cluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:200\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:12.553Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"023f64da-7b35-4990-b404-38d2cd46941a"}
{"level":"error","ts":"2026-07-13T16:14:12.553Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"023f64da-7b35-4990-b404-38d2cd46941a","error":"rayclusters.ray.io \"ray-cluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:14:12.553Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a528c0e6-2c16-4499-aaf2-878289d849c2","namespacedName":{"name":"ray-cluster","namespace":"test-ns-9wtq2"}}
{"level":"info","ts":"2026-07-13T16:14:12.553Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a528c0e6-2c16-4499-aaf2-878289d849c2"}
{"level":"info","ts":"2026-07-13T16:14:12.553Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a528c0e6-2c16-4499-aaf2-878289d849c2","namespace":"test-ns-9wtq2","authClusters":0}
{"level":"info","ts":"2026-07-13T16:14:12.553Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a528c0e6-2c16-4499-aaf2-878289d849c2","namespace":"test-ns-9wtq2"}
{"level":"info","ts":"2026-07-13T16:14:12.559Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"7ae51722-6c29-40e6-9eec-5151fdae7cd4","namespacedName":{"name":"ray-cluster","namespace":"test-ns-9wtq2"}}
{"level":"info","ts":"2026-07-13T16:14:12.559Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"7ae51722-6c29-40e6-9eec-5151fdae7cd4"}
{"level":"info","ts":"2026-07-13T16:14:12.559Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"7ae51722-6c29-40e6-9eec-5151fdae7cd4","namespace":"test-ns-9wtq2","authClusters":0}
{"level":"info","ts":"2026-07-13T16:14:12.559Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"7ae51722-6c29-40e6-9eec-5151fdae7cd4","namespace":"test-ns-9wtq2"}
{"level":"info","ts":"2026-07-13T16:14:12.562Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"2fcd3bce-017d-4548-8284-d00a25e3af32","namespace":"test-ns-9wtq2","clusterName":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.566Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d919750a-af91-4fab-94a2-d6a591357cf4","namespacedName":{"name":"ray-cluster","namespace":"test-ns-9wtq2"}}
{"level":"info","ts":"2026-07-13T16:14:12.566Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d919750a-af91-4fab-94a2-d6a591357cf4"}
{"level":"info","ts":"2026-07-13T16:14:12.566Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d919750a-af91-4fab-94a2-d6a591357cf4","namespace":"test-ns-9wtq2","authClusters":0}
{"level":"info","ts":"2026-07-13T16:14:12.566Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"d919750a-af91-4fab-94a2-d6a591357cf4","namespace":"test-ns-9wtq2"}
{"level":"info","ts":"2026-07-13T16:14:12.567Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a463dd2e-742f-447b-917f-03da428649bf"}
{"level":"info","ts":"2026-07-13T16:14:12.575Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"a0c6dc3e-15bb-476a-bb8c-2d90cccbc79d"}
{"level":"info","ts":"2026-07-13T16:14:12.577Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"07daec6f-8118-49c9-b045-385d99684513","namespacedName":{"name":"ray-cluster","namespace":"test-ns-9wtq2"}}
{"level":"info","ts":"2026-07-13T16:14:12.577Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"07daec6f-8118-49c9-b045-385d99684513"}
{"level":"info","ts":"2026-07-13T16:14:12.577Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"07daec6f-8118-49c9-b045-385d99684513","namespace":"test-ns-9wtq2","authClusters":0}
{"level":"info","ts":"2026-07-13T16:14:12.577Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"07daec6f-8118-49c9-b045-385d99684513","namespace":"test-ns-9wtq2"}
{"level":"info","ts":"2026-07-13T16:14:12.621Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"de05c46e-cd53-4b61-9dfc-906b7dfea64d","namespace":"test-ns-9wtq2","clusterName":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.621Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-9wtq2"},"namespace":"test-ns-9wtq2","name":"ray-cluster","reconcileID":"de05c46e-cd53-4b61-9dfc-906b7dfea64d","namespace":"test-ns-9wtq2","clusterName":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:12.741Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c134f1a2-0888-4134-8f8b-5c4004b62f2d"}
{"level":"info","ts":"2026-07-13T16:14:12.741Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c134f1a2-0888-4134-8f8b-5c4004b62f2d"}
{"level":"info","ts":"2026-07-13T16:14:12.741Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c134f1a2-0888-4134-8f8b-5c4004b62f2d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:12.742Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c134f1a2-0888-4134-8f8b-5c4004b62f2d","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:14:12.742Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c134f1a2-0888-4134-8f8b-5c4004b62f2d","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:12.742Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c134f1a2-0888-4134-8f8b-5c4004b62f2d","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:14:12.742Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c134f1a2-0888-4134-8f8b-5c4004b62f2d","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:12.742Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c134f1a2-0888-4134-8f8b-5c4004b62f2d","worker Pod":"ray-cluster-small-group-worker-cgx8s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-cgx8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:12.742Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c134f1a2-0888-4134-8f8b-5c4004b62f2d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:12.742Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c134f1a2-0888-4134-8f8b-5c4004b62f2d","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:14:12.742Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c134f1a2-0888-4134-8f8b-5c4004b62f2d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:12.742Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c134f1a2-0888-4134-8f8b-5c4004b62f2d","podName":"ray-cluster-small-group-worker-2hvgl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:14:12.742Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c134f1a2-0888-4134-8f8b-5c4004b62f2d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:14:12.742Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c134f1a2-0888-4134-8f8b-5c4004b62f2d","seconds":300}
{"level":"info","ts":"2026-07-13T16:14:13.469Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1361dcc0-9e1e-46b4-9ab1-010d3948e7fe"}
{"level":"info","ts":"2026-07-13T16:14:13.469Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1361dcc0-9e1e-46b4-9ab1-010d3948e7fe"}
{"level":"info","ts":"2026-07-13T16:14:13.469Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1361dcc0-9e1e-46b4-9ab1-010d3948e7fe","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:14:13.470Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1361dcc0-9e1e-46b4-9ab1-010d3948e7fe","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:14:13.470Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1361dcc0-9e1e-46b4-9ab1-010d3948e7fe","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:13.470Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1361dcc0-9e1e-46b4-9ab1-010d3948e7fe","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:14:13.470Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1361dcc0-9e1e-46b4-9ab1-010d3948e7fe","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:13.470Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1361dcc0-9e1e-46b4-9ab1-010d3948e7fe","worker Pod":"ray-cluster-small-group-worker-cgx8s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-cgx8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:14:13.470Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1361dcc0-9e1e-46b4-9ab1-010d3948e7fe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:13.470Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1361dcc0-9e1e-46b4-9ab1-010d3948e7fe","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:14:13.470Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1361dcc0-9e1e-46b4-9ab1-010d3948e7fe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:14:13.470Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1361dcc0-9e1e-46b4-9ab1-010d3948e7fe","podName":"ray-cluster-small-group-worker-2hvgl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:14:13.470Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1361dcc0-9e1e-46b4-9ab1-010d3948e7fe","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:14:13.470Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1361dcc0-9e1e-46b4-9ab1-010d3948e7fe","seconds":300}
{"level":"info","ts":"2026-07-13T16:14:42.327Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"70037fe8-8dff-4b46-bb2f-45c865c0542c","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:14:42.327Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"70037fe8-8dff-4b46-bb2f-45c865c0542c","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:14:42.327Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"70037fe8-8dff-4b46-bb2f-45c865c0542c","pod":"ray-cluster-small-group-worker-2hvgl","ip":"10.133.0.41"}
{"level":"info","ts":"2026-07-13T16:14:42.327Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"70037fe8-8dff-4b46-bb2f-45c865c0542c","pod":"ray-cluster-small-group-worker-cgx8s","ip":"10.133.0.42"}
{"level":"info","ts":"2026-07-13T16:14:42.338Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"70037fe8-8dff-4b46-bb2f-45c865c0542c","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-4j8fw.svc","ray-cluster-head-svc.test-ns-4j8fw.svc.cluster.local"],"ipAddresses":["10.134.0.39","10.133.0.41","10.133.0.42","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:14:42.350Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"70037fe8-8dff-4b46-bb2f-45c865c0542c","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-4j8fw.svc","ray-cluster-worker-svc.test-ns-4j8fw.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-4j8fw.svc","*.ray-cluster-worker-svc.test-ns-4j8fw.svc.cluster.local","*-worker-*.test-ns-4j8fw.svc","*-worker-*.test-ns-4j8fw.svc.cluster.local"],"ipAddresses":["10.134.0.39","10.133.0.41","10.133.0.42","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:14:42.350Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"70037fe8-8dff-4b46-bb2f-45c865c0542c","rayCluster":"ray-cluster","podIPs":["10.134.0.39","10.133.0.41","10.133.0.42"]}
{"level":"info","ts":"2026-07-13T16:15:23.710Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"fecb6b3c-6246-49c2-913c-8483d6a33514"}
{"level":"info","ts":"2026-07-13T16:15:23.710Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"fecb6b3c-6246-49c2-913c-8483d6a33514"}
{"level":"info","ts":"2026-07-13T16:15:23.710Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"fecb6b3c-6246-49c2-913c-8483d6a33514","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:15:23.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"fecb6b3c-6246-49c2-913c-8483d6a33514","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:15:23.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"fecb6b3c-6246-49c2-913c-8483d6a33514","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:23.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"fecb6b3c-6246-49c2-913c-8483d6a33514","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:15:23.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"fecb6b3c-6246-49c2-913c-8483d6a33514","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:23.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"fecb6b3c-6246-49c2-913c-8483d6a33514","worker Pod":"ray-cluster-small-group-worker-cgx8s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-cgx8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:23.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"fecb6b3c-6246-49c2-913c-8483d6a33514","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:23.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"fecb6b3c-6246-49c2-913c-8483d6a33514","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:15:23.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"fecb6b3c-6246-49c2-913c-8483d6a33514","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:23.710Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"fecb6b3c-6246-49c2-913c-8483d6a33514","podName":"ray-cluster-small-group-worker-2hvgl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:15:23.711Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"fecb6b3c-6246-49c2-913c-8483d6a33514","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:15:23.711Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"fecb6b3c-6246-49c2-913c-8483d6a33514","seconds":300}
{"level":"info","ts":"2026-07-13T16:15:23.727Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a49d00f8-a0f1-4413-aac9-9458bae8df09"}
{"level":"info","ts":"2026-07-13T16:15:23.727Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a49d00f8-a0f1-4413-aac9-9458bae8df09"}
{"level":"info","ts":"2026-07-13T16:15:23.727Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a49d00f8-a0f1-4413-aac9-9458bae8df09","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:15:23.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a49d00f8-a0f1-4413-aac9-9458bae8df09","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:15:23.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a49d00f8-a0f1-4413-aac9-9458bae8df09","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:23.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a49d00f8-a0f1-4413-aac9-9458bae8df09","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:15:23.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a49d00f8-a0f1-4413-aac9-9458bae8df09","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:23.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a49d00f8-a0f1-4413-aac9-9458bae8df09","worker Pod":"ray-cluster-small-group-worker-cgx8s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-cgx8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:23.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a49d00f8-a0f1-4413-aac9-9458bae8df09","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:23.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a49d00f8-a0f1-4413-aac9-9458bae8df09","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:15:23.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a49d00f8-a0f1-4413-aac9-9458bae8df09","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:23.727Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a49d00f8-a0f1-4413-aac9-9458bae8df09","podName":"ray-cluster-small-group-worker-2hvgl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:15:23.727Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a49d00f8-a0f1-4413-aac9-9458bae8df09","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:15:23.727Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"a49d00f8-a0f1-4413-aac9-9458bae8df09","seconds":300}
{"level":"info","ts":"2026-07-13T16:15:24.719Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6d943988-61ef-47fd-8d44-f65ebf6968ac"}
{"level":"info","ts":"2026-07-13T16:15:24.719Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6d943988-61ef-47fd-8d44-f65ebf6968ac"}
{"level":"info","ts":"2026-07-13T16:15:24.719Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6d943988-61ef-47fd-8d44-f65ebf6968ac","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:15:24.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6d943988-61ef-47fd-8d44-f65ebf6968ac","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:15:24.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6d943988-61ef-47fd-8d44-f65ebf6968ac","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:24.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6d943988-61ef-47fd-8d44-f65ebf6968ac","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:15:24.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6d943988-61ef-47fd-8d44-f65ebf6968ac","worker Pod":"ray-cluster-small-group-worker-cgx8s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-cgx8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:24.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6d943988-61ef-47fd-8d44-f65ebf6968ac","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:24.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6d943988-61ef-47fd-8d44-f65ebf6968ac","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:24.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6d943988-61ef-47fd-8d44-f65ebf6968ac","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:15:24.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6d943988-61ef-47fd-8d44-f65ebf6968ac","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:24.719Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6d943988-61ef-47fd-8d44-f65ebf6968ac","podName":"ray-cluster-small-group-worker-2hvgl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:15:24.719Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6d943988-61ef-47fd-8d44-f65ebf6968ac","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:15:24.719Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6d943988-61ef-47fd-8d44-f65ebf6968ac","seconds":300}
{"level":"info","ts":"2026-07-13T16:15:24.736Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c08b5fb3-9807-42fe-a7e3-01826605a036"}
{"level":"info","ts":"2026-07-13T16:15:24.736Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c08b5fb3-9807-42fe-a7e3-01826605a036"}
{"level":"info","ts":"2026-07-13T16:15:24.736Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c08b5fb3-9807-42fe-a7e3-01826605a036","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:15:24.736Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c08b5fb3-9807-42fe-a7e3-01826605a036","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:15:24.736Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c08b5fb3-9807-42fe-a7e3-01826605a036","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:24.736Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c08b5fb3-9807-42fe-a7e3-01826605a036","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:15:24.736Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c08b5fb3-9807-42fe-a7e3-01826605a036","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:24.736Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c08b5fb3-9807-42fe-a7e3-01826605a036","worker Pod":"ray-cluster-small-group-worker-cgx8s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-cgx8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:24.736Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c08b5fb3-9807-42fe-a7e3-01826605a036","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:24.736Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c08b5fb3-9807-42fe-a7e3-01826605a036","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:15:24.736Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c08b5fb3-9807-42fe-a7e3-01826605a036","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:24.736Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c08b5fb3-9807-42fe-a7e3-01826605a036","podName":"ray-cluster-small-group-worker-2hvgl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:15:24.737Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c08b5fb3-9807-42fe-a7e3-01826605a036","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:15:24.737Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"c08b5fb3-9807-42fe-a7e3-01826605a036","seconds":300}
{"level":"info","ts":"2026-07-13T16:15:33.756Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"70643798-9818-49f4-9b4a-15c3170c6033"}
{"level":"info","ts":"2026-07-13T16:15:33.756Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"70643798-9818-49f4-9b4a-15c3170c6033"}
{"level":"info","ts":"2026-07-13T16:15:33.756Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"70643798-9818-49f4-9b4a-15c3170c6033","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:15:33.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"70643798-9818-49f4-9b4a-15c3170c6033","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:15:33.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"70643798-9818-49f4-9b4a-15c3170c6033","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:33.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"70643798-9818-49f4-9b4a-15c3170c6033","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:15:33.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"70643798-9818-49f4-9b4a-15c3170c6033","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:33.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"70643798-9818-49f4-9b4a-15c3170c6033","worker Pod":"ray-cluster-small-group-worker-cgx8s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-cgx8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:33.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"70643798-9818-49f4-9b4a-15c3170c6033","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:33.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"70643798-9818-49f4-9b4a-15c3170c6033","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:15:33.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"70643798-9818-49f4-9b4a-15c3170c6033","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:33.756Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"70643798-9818-49f4-9b4a-15c3170c6033","podName":"ray-cluster-small-group-worker-2hvgl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:15:33.756Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"70643798-9818-49f4-9b4a-15c3170c6033","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:15:33.756Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"70643798-9818-49f4-9b4a-15c3170c6033","seconds":300}
{"level":"info","ts":"2026-07-13T16:15:33.778Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8712da6d-dfd4-4d60-976d-d4f361703470"}
{"level":"info","ts":"2026-07-13T16:15:33.778Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8712da6d-dfd4-4d60-976d-d4f361703470"}
{"level":"info","ts":"2026-07-13T16:15:33.778Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8712da6d-dfd4-4d60-976d-d4f361703470","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:15:33.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8712da6d-dfd4-4d60-976d-d4f361703470","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:15:33.779Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8712da6d-dfd4-4d60-976d-d4f361703470","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:33.779Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8712da6d-dfd4-4d60-976d-d4f361703470","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:15:33.779Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8712da6d-dfd4-4d60-976d-d4f361703470","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:33.779Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8712da6d-dfd4-4d60-976d-d4f361703470","worker Pod":"ray-cluster-small-group-worker-cgx8s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-cgx8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:33.779Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8712da6d-dfd4-4d60-976d-d4f361703470","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:33.779Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8712da6d-dfd4-4d60-976d-d4f361703470","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:15:33.779Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8712da6d-dfd4-4d60-976d-d4f361703470","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:33.779Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8712da6d-dfd4-4d60-976d-d4f361703470","podName":"ray-cluster-small-group-worker-cgx8s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-07-13T16:15:33.779Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8712da6d-dfd4-4d60-976d-d4f361703470","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:15:33.779Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8712da6d-dfd4-4d60-976d-d4f361703470","seconds":300}
{"level":"info","ts":"2026-07-13T16:15:34.766Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bacb5914-525e-432a-afd0-2ec60e402522"}
{"level":"info","ts":"2026-07-13T16:15:34.766Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bacb5914-525e-432a-afd0-2ec60e402522"}
{"level":"info","ts":"2026-07-13T16:15:34.766Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bacb5914-525e-432a-afd0-2ec60e402522","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:15:34.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bacb5914-525e-432a-afd0-2ec60e402522","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:15:34.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bacb5914-525e-432a-afd0-2ec60e402522","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:34.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bacb5914-525e-432a-afd0-2ec60e402522","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:15:34.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bacb5914-525e-432a-afd0-2ec60e402522","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:34.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bacb5914-525e-432a-afd0-2ec60e402522","worker Pod":"ray-cluster-small-group-worker-cgx8s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-cgx8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:34.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bacb5914-525e-432a-afd0-2ec60e402522","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:34.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bacb5914-525e-432a-afd0-2ec60e402522","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:15:34.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bacb5914-525e-432a-afd0-2ec60e402522","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:34.767Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bacb5914-525e-432a-afd0-2ec60e402522","podName":"ray-cluster-small-group-worker-cgx8s","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:14:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:15:34.767Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bacb5914-525e-432a-afd0-2ec60e402522","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:15:34.767Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"bacb5914-525e-432a-afd0-2ec60e402522","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:14:12Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:15:34Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-13T16:15:34.779Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"c2c7b634-86ca-40d2-a3ce-f04b3933360a","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:15:34.779Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"4923e318-a278-4b93-87be-cb5228b42b72","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:15:34.779Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"c2c7b634-86ca-40d2-a3ce-f04b3933360a","pod":"ray-cluster-small-group-worker-cgx8s","ip":"10.133.0.42"}
{"level":"info","ts":"2026-07-13T16:15:34.779Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"c2c7b634-86ca-40d2-a3ce-f04b3933360a","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:15:34.779Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"c2c7b634-86ca-40d2-a3ce-f04b3933360a","pod":"ray-cluster-small-group-worker-2hvgl","ip":"10.133.0.41"}
{"level":"info","ts":"2026-07-13T16:15:34.780Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"c2c7b634-86ca-40d2-a3ce-f04b3933360a","rayCluster":"ray-cluster","podIPs":["10.133.0.42","10.134.0.39","10.133.0.41"]}
{"level":"info","ts":"2026-07-13T16:15:34.787Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"715f8835-d102-46a4-b825-6bab3d82ce26"}
{"level":"info","ts":"2026-07-13T16:15:34.787Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"715f8835-d102-46a4-b825-6bab3d82ce26"}
{"level":"info","ts":"2026-07-13T16:15:34.787Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"715f8835-d102-46a4-b825-6bab3d82ce26","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:15:34.787Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"715f8835-d102-46a4-b825-6bab3d82ce26","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:15:34.787Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"715f8835-d102-46a4-b825-6bab3d82ce26","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:34.787Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"715f8835-d102-46a4-b825-6bab3d82ce26","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:15:34.787Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"715f8835-d102-46a4-b825-6bab3d82ce26","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:34.787Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"715f8835-d102-46a4-b825-6bab3d82ce26","worker Pod":"ray-cluster-small-group-worker-cgx8s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-cgx8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:34.787Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"715f8835-d102-46a4-b825-6bab3d82ce26","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:34.787Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"715f8835-d102-46a4-b825-6bab3d82ce26","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:15:34.787Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"715f8835-d102-46a4-b825-6bab3d82ce26","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:34.787Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"715f8835-d102-46a4-b825-6bab3d82ce26","podName":"ray-cluster-small-group-worker-2hvgl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:14:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:15:34.787Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"715f8835-d102-46a4-b825-6bab3d82ce26","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:15:34.787Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"715f8835-d102-46a4-b825-6bab3d82ce26","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:15:34Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:15:34Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-13T16:15:34.797Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"4923e318-a278-4b93-87be-cb5228b42b72","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:15:34.799Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"27395c81-a796-45b2-b1d5-c03c875b18d0","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:15:34.799Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"27395c81-a796-45b2-b1d5-c03c875b18d0","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:15:34.799Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"27395c81-a796-45b2-b1d5-c03c875b18d0","pod":"ray-cluster-small-group-worker-2hvgl","ip":"10.133.0.41"}
{"level":"info","ts":"2026-07-13T16:15:34.799Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"27395c81-a796-45b2-b1d5-c03c875b18d0","pod":"ray-cluster-small-group-worker-cgx8s","ip":"10.133.0.42"}
{"level":"info","ts":"2026-07-13T16:15:34.799Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"27395c81-a796-45b2-b1d5-c03c875b18d0","rayCluster":"ray-cluster","podIPs":["10.134.0.39","10.133.0.41","10.133.0.42"]}
{"level":"info","ts":"2026-07-13T16:15:34.805Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"4923e318-a278-4b93-87be-cb5228b42b72","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:15:34.806Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"4923e318-a278-4b93-87be-cb5228b42b72","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:15:34.806Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"bd1c26ef-1b61-42d5-8b59-9b6283138bc8","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:15:34.825Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"bd1c26ef-1b61-42d5-8b59-9b6283138bc8","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:15:34.834Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"bd1c26ef-1b61-42d5-8b59-9b6283138bc8","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:15:34.834Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"bd1c26ef-1b61-42d5-8b59-9b6283138bc8","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:15:36.779Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0e2e4547-277e-4835-bfcf-881d5097a39c"}
{"level":"info","ts":"2026-07-13T16:15:36.779Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0e2e4547-277e-4835-bfcf-881d5097a39c"}
{"level":"info","ts":"2026-07-13T16:15:36.779Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0e2e4547-277e-4835-bfcf-881d5097a39c","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:15:36.779Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0e2e4547-277e-4835-bfcf-881d5097a39c","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:15:36.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0e2e4547-277e-4835-bfcf-881d5097a39c","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:36.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0e2e4547-277e-4835-bfcf-881d5097a39c","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:15:36.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0e2e4547-277e-4835-bfcf-881d5097a39c","worker Pod":"ray-cluster-small-group-worker-cgx8s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-cgx8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:36.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0e2e4547-277e-4835-bfcf-881d5097a39c","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:36.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0e2e4547-277e-4835-bfcf-881d5097a39c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:36.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0e2e4547-277e-4835-bfcf-881d5097a39c","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:15:36.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0e2e4547-277e-4835-bfcf-881d5097a39c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:36.780Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0e2e4547-277e-4835-bfcf-881d5097a39c","podName":"ray-cluster-small-group-worker-2hvgl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:14:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:15:36.780Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0e2e4547-277e-4835-bfcf-881d5097a39c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:15:36.780Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0e2e4547-277e-4835-bfcf-881d5097a39c","seconds":300}
{"level":"info","ts":"2026-07-13T16:15:42.351Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"aa166cb5-c38b-47fd-8dd0-eee094516704","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:15:42.351Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"aa166cb5-c38b-47fd-8dd0-eee094516704","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:15:42.351Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"aa166cb5-c38b-47fd-8dd0-eee094516704","pod":"ray-cluster-small-group-worker-2hvgl","ip":"10.133.0.41"}
{"level":"info","ts":"2026-07-13T16:15:42.351Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"aa166cb5-c38b-47fd-8dd0-eee094516704","pod":"ray-cluster-small-group-worker-cgx8s","ip":"10.133.0.42"}
{"level":"info","ts":"2026-07-13T16:15:42.351Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"aa166cb5-c38b-47fd-8dd0-eee094516704","rayCluster":"ray-cluster","podIPs":["10.134.0.39","10.133.0.41","10.133.0.42"]}
{"level":"info","ts":"2026-07-13T16:15:45.790Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ba85f630-4a1e-4ff7-9dbe-21dc1f55e064"}
{"level":"info","ts":"2026-07-13T16:15:45.790Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ba85f630-4a1e-4ff7-9dbe-21dc1f55e064"}
{"level":"info","ts":"2026-07-13T16:15:45.790Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ba85f630-4a1e-4ff7-9dbe-21dc1f55e064","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:15:45.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ba85f630-4a1e-4ff7-9dbe-21dc1f55e064","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:15:45.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ba85f630-4a1e-4ff7-9dbe-21dc1f55e064","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:45.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ba85f630-4a1e-4ff7-9dbe-21dc1f55e064","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:15:45.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ba85f630-4a1e-4ff7-9dbe-21dc1f55e064","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:45.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ba85f630-4a1e-4ff7-9dbe-21dc1f55e064","worker Pod":"ray-cluster-small-group-worker-cgx8s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-cgx8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:45.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ba85f630-4a1e-4ff7-9dbe-21dc1f55e064","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:45.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ba85f630-4a1e-4ff7-9dbe-21dc1f55e064","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:15:45.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ba85f630-4a1e-4ff7-9dbe-21dc1f55e064","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:45.791Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ba85f630-4a1e-4ff7-9dbe-21dc1f55e064","podName":"ray-cluster-small-group-worker-2hvgl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-07-13T16:14:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-07-13T16:15:45.791Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ba85f630-4a1e-4ff7-9dbe-21dc1f55e064","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:15:45.791Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"ba85f630-4a1e-4ff7-9dbe-21dc1f55e064","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:15:34Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:15:45Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-13T16:15:45.802Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5b41e946-a6cc-4ead-864d-8065a7134a1d","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:15:45.802Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"2e580f1e-a1ba-43f5-902d-0a3ca243937b","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:15:45.802Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5b41e946-a6cc-4ead-864d-8065a7134a1d","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:15:45.802Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5b41e946-a6cc-4ead-864d-8065a7134a1d","pod":"ray-cluster-small-group-worker-2hvgl","ip":"10.133.0.41"}
{"level":"info","ts":"2026-07-13T16:15:45.802Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5b41e946-a6cc-4ead-864d-8065a7134a1d","pod":"ray-cluster-small-group-worker-cgx8s","ip":"10.133.0.42"}
{"level":"info","ts":"2026-07-13T16:15:45.802Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5b41e946-a6cc-4ead-864d-8065a7134a1d","rayCluster":"ray-cluster","podIPs":["10.134.0.39","10.133.0.41","10.133.0.42"]}
{"level":"info","ts":"2026-07-13T16:15:45.808Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"11c6778b-d26d-4789-a93d-515fb6dba7dc"}
{"level":"info","ts":"2026-07-13T16:15:45.808Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"11c6778b-d26d-4789-a93d-515fb6dba7dc"}
{"level":"info","ts":"2026-07-13T16:15:45.808Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"11c6778b-d26d-4789-a93d-515fb6dba7dc","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:15:45.808Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"11c6778b-d26d-4789-a93d-515fb6dba7dc","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:15:45.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"11c6778b-d26d-4789-a93d-515fb6dba7dc","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:45.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"11c6778b-d26d-4789-a93d-515fb6dba7dc","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:15:45.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"11c6778b-d26d-4789-a93d-515fb6dba7dc","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:45.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"11c6778b-d26d-4789-a93d-515fb6dba7dc","worker Pod":"ray-cluster-small-group-worker-cgx8s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-cgx8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:45.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"11c6778b-d26d-4789-a93d-515fb6dba7dc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:45.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"11c6778b-d26d-4789-a93d-515fb6dba7dc","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:15:45.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"11c6778b-d26d-4789-a93d-515fb6dba7dc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:45.809Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"11c6778b-d26d-4789-a93d-515fb6dba7dc","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":2,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":2,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:15:45.809Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"11c6778b-d26d-4789-a93d-515fb6dba7dc","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:15:45Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:15:45Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-07-13T16:15:45.821Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"2e580f1e-a1ba-43f5-902d-0a3ca243937b","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:15:45.823Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"aaef6f31-e774-4a10-8879-284a25bd8216","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:15:45.823Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"aaef6f31-e774-4a10-8879-284a25bd8216","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:15:45.823Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"aaef6f31-e774-4a10-8879-284a25bd8216","pod":"ray-cluster-small-group-worker-2hvgl","ip":"10.133.0.41"}
{"level":"info","ts":"2026-07-13T16:15:45.823Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"aaef6f31-e774-4a10-8879-284a25bd8216","pod":"ray-cluster-small-group-worker-cgx8s","ip":"10.133.0.42"}
{"level":"info","ts":"2026-07-13T16:15:45.823Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"aaef6f31-e774-4a10-8879-284a25bd8216","rayCluster":"ray-cluster","podIPs":["10.134.0.39","10.133.0.41","10.133.0.42"]}
{"level":"info","ts":"2026-07-13T16:15:45.829Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"2e580f1e-a1ba-43f5-902d-0a3ca243937b","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:15:45.829Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"2e580f1e-a1ba-43f5-902d-0a3ca243937b","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:15:45.829Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"8312ccae-42f2-4a80-b26d-10137a90e23b","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:15:45.837Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"8312ccae-42f2-4a80-b26d-10137a90e23b","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:15:45.847Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"8312ccae-42f2-4a80-b26d-10137a90e23b","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:15:45.847Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"8312ccae-42f2-4a80-b26d-10137a90e23b","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:15:47.803Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"37a5dd63-81c9-46b5-83bf-5f10ed8f4682"}
{"level":"info","ts":"2026-07-13T16:15:47.803Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"37a5dd63-81c9-46b5-83bf-5f10ed8f4682"}
{"level":"info","ts":"2026-07-13T16:15:47.803Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"37a5dd63-81c9-46b5-83bf-5f10ed8f4682","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:15:47.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"37a5dd63-81c9-46b5-83bf-5f10ed8f4682","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:15:47.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"37a5dd63-81c9-46b5-83bf-5f10ed8f4682","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:47.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"37a5dd63-81c9-46b5-83bf-5f10ed8f4682","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-07-13T16:15:47.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"37a5dd63-81c9-46b5-83bf-5f10ed8f4682","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:47.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"37a5dd63-81c9-46b5-83bf-5f10ed8f4682","worker Pod":"ray-cluster-small-group-worker-cgx8s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-cgx8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:15:47.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"37a5dd63-81c9-46b5-83bf-5f10ed8f4682","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:47.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"37a5dd63-81c9-46b5-83bf-5f10ed8f4682","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-07-13T16:15:47.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"37a5dd63-81c9-46b5-83bf-5f10ed8f4682","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:15:47.803Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"37a5dd63-81c9-46b5-83bf-5f10ed8f4682","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:15:47.803Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"37a5dd63-81c9-46b5-83bf-5f10ed8f4682","seconds":300}
{"level":"info","ts":"2026-07-13T16:16:42.352Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"84506a54-3d73-4321-85ed-755dc11b0eed","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:16:42.352Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"84506a54-3d73-4321-85ed-755dc11b0eed","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:16:42.352Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"84506a54-3d73-4321-85ed-755dc11b0eed","pod":"ray-cluster-small-group-worker-2hvgl","ip":"10.133.0.41"}
{"level":"info","ts":"2026-07-13T16:16:42.352Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"84506a54-3d73-4321-85ed-755dc11b0eed","pod":"ray-cluster-small-group-worker-cgx8s","ip":"10.133.0.42"}
{"level":"info","ts":"2026-07-13T16:16:42.352Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"84506a54-3d73-4321-85ed-755dc11b0eed","rayCluster":"ray-cluster","podIPs":["10.134.0.39","10.133.0.41","10.133.0.42"]}
{"level":"info","ts":"2026-07-13T16:16:44.301Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:16:44.301Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-4j8fw"}
{"level":"info","ts":"2026-07-13T16:16:44.312Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"443f92c0-e28d-46fc-b8ae-0a2e653415ae","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:16:44.312Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"ff8f38c0-7a80-486a-a392-b2c19ac5b4db","namespacedName":{"name":"ray-cluster","namespace":"test-ns-4j8fw"}}
{"level":"info","ts":"2026-07-13T16:16:44.312Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"43011a0f-96e9-43c7-b9c2-0ecafe72260d","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:16:44.312Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"ff8f38c0-7a80-486a-a392-b2c19ac5b4db","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:16:44.312Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"ff8f38c0-7a80-486a-a392-b2c19ac5b4db","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:16:44.312Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06173398-741d-40eb-a3a1-266d5008e6b2"}
{"level":"info","ts":"2026-07-13T16:16:44.312Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06173398-741d-40eb-a3a1-266d5008e6b2"}
{"level":"info","ts":"2026-07-13T16:16:44.312Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"43011a0f-96e9-43c7-b9c2-0ecafe72260d","pod":"ray-cluster-small-group-worker-2hvgl","ip":"10.133.0.41"}
{"level":"info","ts":"2026-07-13T16:16:44.312Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"43011a0f-96e9-43c7-b9c2-0ecafe72260d","pod":"ray-cluster-small-group-worker-cgx8s","ip":"10.133.0.42"}
{"level":"info","ts":"2026-07-13T16:16:44.312Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"43011a0f-96e9-43c7-b9c2-0ecafe72260d","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:16:44.312Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"43011a0f-96e9-43c7-b9c2-0ecafe72260d","rayCluster":"ray-cluster","podIPs":["10.133.0.41","10.133.0.42","10.134.0.39"]}
{"level":"info","ts":"2026-07-13T16:16:44.312Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06173398-741d-40eb-a3a1-266d5008e6b2","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:16:44.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06173398-741d-40eb-a3a1-266d5008e6b2","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:16:44.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06173398-741d-40eb-a3a1-266d5008e6b2","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:16:44.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06173398-741d-40eb-a3a1-266d5008e6b2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-13T16:16:44.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06173398-741d-40eb-a3a1-266d5008e6b2","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:16:44.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06173398-741d-40eb-a3a1-266d5008e6b2","worker Pod":"ray-cluster-small-group-worker-cgx8s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-cgx8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:16:44.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06173398-741d-40eb-a3a1-266d5008e6b2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:16:44.312Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06173398-741d-40eb-a3a1-266d5008e6b2","namespace":"test-ns-4j8fw","name":"ray-cluster-small-group-worker-cgx8s"}
{"level":"info","ts":"2026-07-13T16:16:44.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06173398-741d-40eb-a3a1-266d5008e6b2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:16:44.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06173398-741d-40eb-a3a1-266d5008e6b2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:16:44.321Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06173398-741d-40eb-a3a1-266d5008e6b2","oldReadyWorkerReplicas":2,"newReadyWorkerReplicas":2,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:16:44.321Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06173398-741d-40eb-a3a1-266d5008e6b2","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:15:45Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:16:44Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-07-13T16:16:44.324Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"ff8f38c0-7a80-486a-a392-b2c19ac5b4db","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:16:44.324Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"172ec166-4814-42e5-98d4-0b84c7f914d9","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:16:44.324Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"172ec166-4814-42e5-98d4-0b84c7f914d9","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:16:44.324Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"172ec166-4814-42e5-98d4-0b84c7f914d9","pod":"ray-cluster-small-group-worker-2hvgl","ip":"10.133.0.41"}
{"level":"info","ts":"2026-07-13T16:16:44.324Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"172ec166-4814-42e5-98d4-0b84c7f914d9","pod":"ray-cluster-small-group-worker-cgx8s","ip":"10.133.0.42"}
{"level":"info","ts":"2026-07-13T16:16:44.324Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"172ec166-4814-42e5-98d4-0b84c7f914d9","rayCluster":"ray-cluster","podIPs":["10.134.0.39","10.133.0.41","10.133.0.42"]}
{"level":"info","ts":"2026-07-13T16:16:44.329Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"443f92c0-e28d-46fc-b8ae-0a2e653415ae","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:16:44.330Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06173398-741d-40eb-a3a1-266d5008e6b2","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4j8fw","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-13T16:16:44.330Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06173398-741d-40eb-a3a1-266d5008e6b2"}
{"level":"error","ts":"2026-07-13T16:16:44.330Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06173398-741d-40eb-a3a1-266d5008e6b2","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:16:44.330Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1b8eff0c-f8b5-4017-8177-11a9d0987504"}
{"level":"info","ts":"2026-07-13T16:16:44.330Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1b8eff0c-f8b5-4017-8177-11a9d0987504"}
{"level":"info","ts":"2026-07-13T16:16:44.330Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1b8eff0c-f8b5-4017-8177-11a9d0987504","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:16:44.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1b8eff0c-f8b5-4017-8177-11a9d0987504","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:16:44.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1b8eff0c-f8b5-4017-8177-11a9d0987504","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:16:44.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1b8eff0c-f8b5-4017-8177-11a9d0987504","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:16:44.330Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1b8eff0c-f8b5-4017-8177-11a9d0987504","oldReadyWorkerReplicas":2,"newReadyWorkerReplicas":2,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:16:44.330Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"1b8eff0c-f8b5-4017-8177-11a9d0987504","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:15:45Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:16:44Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-07-13T16:16:44.340Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"443f92c0-e28d-46fc-b8ae-0a2e653415ae","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:16:44.340Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"443f92c0-e28d-46fc-b8ae-0a2e653415ae","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:16:44.340Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"64ddd694-7b25-4325-a958-7db516fd42c4","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:16:44.341Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06f91795-15c4-4af3-ab7c-7c041586f20b"}
{"level":"info","ts":"2026-07-13T16:16:44.341Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06f91795-15c4-4af3-ab7c-7c041586f20b"}
{"level":"info","ts":"2026-07-13T16:16:44.341Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06f91795-15c4-4af3-ab7c-7c041586f20b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:16:44.341Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06f91795-15c4-4af3-ab7c-7c041586f20b","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:16:44.341Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06f91795-15c4-4af3-ab7c-7c041586f20b","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:16:44.341Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06f91795-15c4-4af3-ab7c-7c041586f20b","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:16:44.341Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"caf79dc9-7355-4d48-b4f5-d17732f544d1","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:16:44.341Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06f91795-15c4-4af3-ab7c-7c041586f20b","oldReadyWorkerReplicas":2,"newReadyWorkerReplicas":2,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":2,"oldDesiredWorkerReplicas":2,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:16:44.341Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06f91795-15c4-4af3-ab7c-7c041586f20b","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:15:45Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:16:44Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-07-13T16:16:44.341Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"caf79dc9-7355-4d48-b4f5-d17732f544d1","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:16:44.341Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"caf79dc9-7355-4d48-b4f5-d17732f544d1","pod":"ray-cluster-small-group-worker-2hvgl","ip":"10.133.0.41"}
{"level":"info","ts":"2026-07-13T16:16:44.341Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"caf79dc9-7355-4d48-b4f5-d17732f544d1","pod":"ray-cluster-small-group-worker-cgx8s","ip":"10.133.0.42"}
{"level":"info","ts":"2026-07-13T16:16:44.341Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"caf79dc9-7355-4d48-b4f5-d17732f544d1","rayCluster":"ray-cluster","podIPs":["10.134.0.39","10.133.0.41","10.133.0.42"]}
{"level":"info","ts":"2026-07-13T16:16:44.421Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06f91795-15c4-4af3-ab7c-7c041586f20b","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4j8fw","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-13T16:16:44.421Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06f91795-15c4-4af3-ab7c-7c041586f20b"}
{"level":"error","ts":"2026-07-13T16:16:44.421Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"06f91795-15c4-4af3-ab7c-7c041586f20b","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:16:44.421Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"eb36b247-e26d-4a71-9f70-ce4219de9e63"}
{"level":"info","ts":"2026-07-13T16:16:44.421Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"eb36b247-e26d-4a71-9f70-ce4219de9e63"}
{"level":"info","ts":"2026-07-13T16:16:44.421Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"eb36b247-e26d-4a71-9f70-ce4219de9e63","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:16:44.422Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"eb36b247-e26d-4a71-9f70-ce4219de9e63","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:16:44.422Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"eb36b247-e26d-4a71-9f70-ce4219de9e63","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:16:44.422Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"eb36b247-e26d-4a71-9f70-ce4219de9e63","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:16:44.422Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"eb36b247-e26d-4a71-9f70-ce4219de9e63","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:16:44.422Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"eb36b247-e26d-4a71-9f70-ce4219de9e63","seconds":300}
{"level":"info","ts":"2026-07-13T16:16:44.425Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"64ddd694-7b25-4325-a958-7db516fd42c4","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:16:44.427Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"f08228ef-f04f-491e-8bdc-5e7e8283df8a"}
{"level":"info","ts":"2026-07-13T16:16:44.427Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"f08228ef-f04f-491e-8bdc-5e7e8283df8a"}
{"level":"info","ts":"2026-07-13T16:16:44.427Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"f08228ef-f04f-491e-8bdc-5e7e8283df8a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:16:44.427Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"f08228ef-f04f-491e-8bdc-5e7e8283df8a","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:16:44.427Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"f08228ef-f04f-491e-8bdc-5e7e8283df8a","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:16:44.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"f08228ef-f04f-491e-8bdc-5e7e8283df8a","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:16:44.428Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"f08228ef-f04f-491e-8bdc-5e7e8283df8a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:16:44.428Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"f08228ef-f04f-491e-8bdc-5e7e8283df8a","seconds":300}
{"level":"info","ts":"2026-07-13T16:16:44.436Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"64ddd694-7b25-4325-a958-7db516fd42c4","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:16:44.436Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"64ddd694-7b25-4325-a958-7db516fd42c4","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:16:44.436Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"904978e2-54e2-40de-90b4-8b242536701c","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:16:44.444Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"904978e2-54e2-40de-90b4-8b242536701c","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:16:44.452Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"904978e2-54e2-40de-90b4-8b242536701c","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:16:44.452Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"904978e2-54e2-40de-90b4-8b242536701c","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:16:44.785Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0da64865-ca96-456a-906c-6c1004fda9e8"}
{"level":"info","ts":"2026-07-13T16:16:44.785Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0da64865-ca96-456a-906c-6c1004fda9e8"}
{"level":"info","ts":"2026-07-13T16:16:44.786Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0da64865-ca96-456a-906c-6c1004fda9e8","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:16:44.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0da64865-ca96-456a-906c-6c1004fda9e8","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:16:44.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0da64865-ca96-456a-906c-6c1004fda9e8","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:16:44.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0da64865-ca96-456a-906c-6c1004fda9e8","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:16:44.786Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0da64865-ca96-456a-906c-6c1004fda9e8","oldReadyWorkerReplicas":2,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":2,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:16:44.786Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"0da64865-ca96-456a-906c-6c1004fda9e8","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:16:44Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:16:44Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-07-13T16:16:44.800Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5b161ff0-7772-4578-9501-b818dad04981","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:16:44.800Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"4c6bcec4-ff80-46f3-989c-f96ed15af9e0","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:16:44.800Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"4c6bcec4-ff80-46f3-989c-f96ed15af9e0","pod":"ray-cluster-small-group-worker-2hvgl","ip":"10.133.0.41"}
{"level":"info","ts":"2026-07-13T16:16:44.800Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"4c6bcec4-ff80-46f3-989c-f96ed15af9e0","pod":"ray-cluster-small-group-worker-cgx8s","ip":"10.133.0.42"}
{"level":"info","ts":"2026-07-13T16:16:44.800Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"4c6bcec4-ff80-46f3-989c-f96ed15af9e0","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:16:44.801Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"4c6bcec4-ff80-46f3-989c-f96ed15af9e0","rayCluster":"ray-cluster","podIPs":["10.133.0.41","10.133.0.42","10.134.0.39"]}
{"level":"info","ts":"2026-07-13T16:16:44.811Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5b161ff0-7772-4578-9501-b818dad04981","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:16:44.824Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5b161ff0-7772-4578-9501-b818dad04981","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:16:44.824Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5b161ff0-7772-4578-9501-b818dad04981","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:16:45.020Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6bdba346-122c-4ff3-9d85-2687e26a8535"}
{"level":"info","ts":"2026-07-13T16:16:45.020Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6bdba346-122c-4ff3-9d85-2687e26a8535"}
{"level":"info","ts":"2026-07-13T16:16:45.020Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6bdba346-122c-4ff3-9d85-2687e26a8535","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:16:45.020Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6bdba346-122c-4ff3-9d85-2687e26a8535","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:16:45.020Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6bdba346-122c-4ff3-9d85-2687e26a8535","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:16:45.020Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6bdba346-122c-4ff3-9d85-2687e26a8535","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:16:45.020Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6bdba346-122c-4ff3-9d85-2687e26a8535","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:16:45.020Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"6bdba346-122c-4ff3-9d85-2687e26a8535","seconds":300}
{"level":"info","ts":"2026-07-13T16:16:45.026Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d17cb62e-a929-431a-84a9-1bf6a8e47b42"}
{"level":"info","ts":"2026-07-13T16:16:45.026Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d17cb62e-a929-431a-84a9-1bf6a8e47b42"}
{"level":"info","ts":"2026-07-13T16:16:45.026Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d17cb62e-a929-431a-84a9-1bf6a8e47b42","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:16:45.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d17cb62e-a929-431a-84a9-1bf6a8e47b42","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:16:45.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d17cb62e-a929-431a-84a9-1bf6a8e47b42","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:16:45.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d17cb62e-a929-431a-84a9-1bf6a8e47b42","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-13T16:16:45.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d17cb62e-a929-431a-84a9-1bf6a8e47b42","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:16:45.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d17cb62e-a929-431a-84a9-1bf6a8e47b42","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:16:45.026Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d17cb62e-a929-431a-84a9-1bf6a8e47b42","namespace":"test-ns-4j8fw","name":"ray-cluster-small-group-worker-cgx8s"}
{"level":"info","ts":"2026-07-13T16:16:45.029Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d17cb62e-a929-431a-84a9-1bf6a8e47b42","The worker Pod has already been deleted":"ray-cluster-small-group-worker-cgx8s"}
{"level":"info","ts":"2026-07-13T16:16:45.029Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d17cb62e-a929-431a-84a9-1bf6a8e47b42","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:16:45.029Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d17cb62e-a929-431a-84a9-1bf6a8e47b42","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:16:45.029Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d17cb62e-a929-431a-84a9-1bf6a8e47b42","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:16:45.029Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"d17cb62e-a929-431a-84a9-1bf6a8e47b42","seconds":300}
{"level":"info","ts":"2026-07-13T16:16:46.801Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8a7a8418-446f-471a-859a-aebfcd660b19"}
{"level":"info","ts":"2026-07-13T16:16:46.801Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8a7a8418-446f-471a-859a-aebfcd660b19"}
{"level":"info","ts":"2026-07-13T16:16:46.801Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8a7a8418-446f-471a-859a-aebfcd660b19","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:16:46.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8a7a8418-446f-471a-859a-aebfcd660b19","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:16:46.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8a7a8418-446f-471a-859a-aebfcd660b19","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:16:46.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8a7a8418-446f-471a-859a-aebfcd660b19","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-07-13T16:16:46.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8a7a8418-446f-471a-859a-aebfcd660b19","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:16:46.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8a7a8418-446f-471a-859a-aebfcd660b19","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:16:46.801Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8a7a8418-446f-471a-859a-aebfcd660b19","namespace":"test-ns-4j8fw","name":"ray-cluster-small-group-worker-cgx8s"}
{"level":"info","ts":"2026-07-13T16:16:46.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8a7a8418-446f-471a-859a-aebfcd660b19","The worker Pod has already been deleted":"ray-cluster-small-group-worker-cgx8s"}
{"level":"info","ts":"2026-07-13T16:16:46.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8a7a8418-446f-471a-859a-aebfcd660b19","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-07-13T16:16:46.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8a7a8418-446f-471a-859a-aebfcd660b19","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:16:46.804Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8a7a8418-446f-471a-859a-aebfcd660b19","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:16:46.804Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"8a7a8418-446f-471a-859a-aebfcd660b19","seconds":300}
{"level":"info","ts":"2026-07-13T16:17:42.352Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"07ef7b5c-0b8a-4b46-8cfd-c65c60c7f0ac","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:17:42.352Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"07ef7b5c-0b8a-4b46-8cfd-c65c60c7f0ac","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:17:42.352Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"07ef7b5c-0b8a-4b46-8cfd-c65c60c7f0ac","pod":"ray-cluster-small-group-worker-2hvgl","ip":"10.133.0.41"}
{"level":"info","ts":"2026-07-13T16:17:42.377Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"07ef7b5c-0b8a-4b46-8cfd-c65c60c7f0ac","certificate":"ray-head-cert-ray-cluster","dnsNames":["ray-cluster-head-svc","localhost","ray-cluster-head-svc.test-ns-4j8fw.svc","ray-cluster-head-svc.test-ns-4j8fw.svc.cluster.local"],"ipAddresses":["10.134.0.39","10.133.0.41","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:17:42.389Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"07ef7b5c-0b8a-4b46-8cfd-c65c60c7f0ac","certificate":"ray-worker-cert-ray-cluster","dnsNames":["ray-cluster-worker-svc","localhost","ray-cluster-small-group","ray-cluster-worker-svc.test-ns-4j8fw.svc","ray-cluster-worker-svc.test-ns-4j8fw.svc.cluster.local","*.ray-cluster-worker-svc.test-ns-4j8fw.svc","*.ray-cluster-worker-svc.test-ns-4j8fw.svc.cluster.local","*-worker-*.test-ns-4j8fw.svc","*-worker-*.test-ns-4j8fw.svc.cluster.local"],"ipAddresses":["10.134.0.39","10.133.0.41","127.0.0.1"]}
{"level":"info","ts":"2026-07-13T16:17:42.389Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"07ef7b5c-0b8a-4b46-8cfd-c65c60c7f0ac","rayCluster":"ray-cluster","podIPs":["10.134.0.39","10.133.0.41"]}
{"level":"info","ts":"2026-07-13T16:17:55.258Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:17:55.258Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"ray-cluster","namespace":"test-ns-4j8fw"}
{"level":"info","ts":"2026-07-13T16:17:55.268Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5075c8c4-2238-4765-986f-c511d1ad97c4","namespacedName":{"name":"ray-cluster","namespace":"test-ns-4j8fw"}}
{"level":"info","ts":"2026-07-13T16:17:55.268Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"591ffc3c-5b84-41b6-bbc8-2559dc9fe504"}
{"level":"info","ts":"2026-07-13T16:17:55.268Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"591ffc3c-5b84-41b6-bbc8-2559dc9fe504"}
{"level":"info","ts":"2026-07-13T16:17:55.268Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"20f284e0-419c-428f-badf-91e465997afe","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:17:55.268Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5075c8c4-2238-4765-986f-c511d1ad97c4","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:17:55.268Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5075c8c4-2238-4765-986f-c511d1ad97c4","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:17:55.268Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"c80dd07d-ac31-468d-b0c0-e48d23cb9679","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:17:55.268Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"591ffc3c-5b84-41b6-bbc8-2559dc9fe504","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:17:55.268Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"c80dd07d-ac31-468d-b0c0-e48d23cb9679","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:17:55.268Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"c80dd07d-ac31-468d-b0c0-e48d23cb9679","pod":"ray-cluster-small-group-worker-2hvgl","ip":"10.133.0.41"}
{"level":"info","ts":"2026-07-13T16:17:55.268Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"591ffc3c-5b84-41b6-bbc8-2559dc9fe504","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:17:55.268Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"591ffc3c-5b84-41b6-bbc8-2559dc9fe504","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:17:55.268Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"591ffc3c-5b84-41b6-bbc8-2559dc9fe504","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:17:55.268Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"591ffc3c-5b84-41b6-bbc8-2559dc9fe504","worker Pod":"ray-cluster-small-group-worker-2hvgl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-2hvgl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:17:55.268Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"591ffc3c-5b84-41b6-bbc8-2559dc9fe504","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:17:55.268Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"591ffc3c-5b84-41b6-bbc8-2559dc9fe504","namespace":"test-ns-4j8fw","name":"ray-cluster-small-group-worker-2hvgl"}
{"level":"info","ts":"2026-07-13T16:17:55.268Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"c80dd07d-ac31-468d-b0c0-e48d23cb9679","rayCluster":"ray-cluster","podIPs":["10.134.0.39","10.133.0.41"]}
{"level":"info","ts":"2026-07-13T16:17:55.277Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"591ffc3c-5b84-41b6-bbc8-2559dc9fe504","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:17:55.277Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"591ffc3c-5b84-41b6-bbc8-2559dc9fe504","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:17:55.277Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"591ffc3c-5b84-41b6-bbc8-2559dc9fe504","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:17:55.277Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"591ffc3c-5b84-41b6-bbc8-2559dc9fe504","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:16:44Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:17:55Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-07-13T16:17:55.280Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5075c8c4-2238-4765-986f-c511d1ad97c4","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:17:55.280Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"20f284e0-419c-428f-badf-91e465997afe","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:17:55.281Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"1b75fe69-5821-4ac7-ae72-16fcc650a6fd","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:17:55.321Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"1b75fe69-5821-4ac7-ae72-16fcc650a6fd","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:17:55.321Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"1b75fe69-5821-4ac7-ae72-16fcc650a6fd","pod":"ray-cluster-small-group-worker-2hvgl","ip":"10.133.0.41"}
{"level":"info","ts":"2026-07-13T16:17:55.321Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"1b75fe69-5821-4ac7-ae72-16fcc650a6fd","rayCluster":"ray-cluster","podIPs":["10.134.0.39","10.133.0.41"]}
{"level":"info","ts":"2026-07-13T16:17:55.321Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"591ffc3c-5b84-41b6-bbc8-2559dc9fe504","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4j8fw","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-13T16:17:55.321Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"591ffc3c-5b84-41b6-bbc8-2559dc9fe504"}
{"level":"error","ts":"2026-07-13T16:17:55.321Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"591ffc3c-5b84-41b6-bbc8-2559dc9fe504","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:17:55.321Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"44e1927a-883e-41d7-925b-662a5b21c8b6"}
{"level":"info","ts":"2026-07-13T16:17:55.321Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"44e1927a-883e-41d7-925b-662a5b21c8b6"}
{"level":"info","ts":"2026-07-13T16:17:55.321Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"44e1927a-883e-41d7-925b-662a5b21c8b6","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:17:55.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"44e1927a-883e-41d7-925b-662a5b21c8b6","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:17:55.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"44e1927a-883e-41d7-925b-662a5b21c8b6","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:17:55.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"44e1927a-883e-41d7-925b-662a5b21c8b6","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:17:55.322Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"44e1927a-883e-41d7-925b-662a5b21c8b6","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:17:55.322Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"44e1927a-883e-41d7-925b-662a5b21c8b6","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:16:44Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:17:55Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-07-13T16:17:55.325Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"20f284e0-419c-428f-badf-91e465997afe","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:17:55.325Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"20f284e0-419c-428f-badf-91e465997afe","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:17:55.325Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"848865bb-f2f4-4e03-85c5-f7642db8c5a1","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:17:55.333Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"091eaab0-2782-47f1-87ab-ec2cc6fe34b9"}
{"level":"info","ts":"2026-07-13T16:17:55.333Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"091eaab0-2782-47f1-87ab-ec2cc6fe34b9"}
{"level":"info","ts":"2026-07-13T16:17:55.333Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"091eaab0-2782-47f1-87ab-ec2cc6fe34b9","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:17:55.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"091eaab0-2782-47f1-87ab-ec2cc6fe34b9","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:17:55.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"091eaab0-2782-47f1-87ab-ec2cc6fe34b9","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:17:55.333Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"091eaab0-2782-47f1-87ab-ec2cc6fe34b9","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:17:55.333Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"091eaab0-2782-47f1-87ab-ec2cc6fe34b9","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:17:55.333Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"091eaab0-2782-47f1-87ab-ec2cc6fe34b9","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:16:44Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:17:55Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-07-13T16:17:55.334Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5c713290-d6cb-4e5d-bb91-455fe749136f","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:17:55.334Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5c713290-d6cb-4e5d-bb91-455fe749136f","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:17:55.334Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5c713290-d6cb-4e5d-bb91-455fe749136f","pod":"ray-cluster-small-group-worker-2hvgl","ip":"10.133.0.41"}
{"level":"info","ts":"2026-07-13T16:17:55.334Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"5c713290-d6cb-4e5d-bb91-455fe749136f","rayCluster":"ray-cluster","podIPs":["10.134.0.39","10.133.0.41"]}
{"level":"info","ts":"2026-07-13T16:17:55.337Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"848865bb-f2f4-4e03-85c5-f7642db8c5a1","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:17:55.339Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"091eaab0-2782-47f1-87ab-ec2cc6fe34b9","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4j8fw","name":"ray-cluster"}}
{"level":"info","ts":"2026-07-13T16:17:55.339Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"091eaab0-2782-47f1-87ab-ec2cc6fe34b9"}
{"level":"error","ts":"2026-07-13T16:17:55.339Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"091eaab0-2782-47f1-87ab-ec2cc6fe34b9","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T16:17:55.344Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"b559400f-bd0a-404a-abc2-5a97177a5ec9"}
{"level":"info","ts":"2026-07-13T16:17:55.344Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"b559400f-bd0a-404a-abc2-5a97177a5ec9"}
{"level":"info","ts":"2026-07-13T16:17:55.344Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"b559400f-bd0a-404a-abc2-5a97177a5ec9","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:17:55.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"b559400f-bd0a-404a-abc2-5a97177a5ec9","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:17:55.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"b559400f-bd0a-404a-abc2-5a97177a5ec9","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:17:55.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"b559400f-bd0a-404a-abc2-5a97177a5ec9","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:17:55.344Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"848865bb-f2f4-4e03-85c5-f7642db8c5a1","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:17:55.344Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"848865bb-f2f4-4e03-85c5-f7642db8c5a1","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:17:55.344Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"0eeb3b31-137a-4a9b-8ad3-e286d4947718","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:17:55.344Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"b559400f-bd0a-404a-abc2-5a97177a5ec9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:17:55.344Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"b559400f-bd0a-404a-abc2-5a97177a5ec9","seconds":300}
{"level":"info","ts":"2026-07-13T16:17:55.353Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"0eeb3b31-137a-4a9b-8ad3-e286d4947718","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:17:55.364Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"0eeb3b31-137a-4a9b-8ad3-e286d4947718","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:17:55.364Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"0eeb3b31-137a-4a9b-8ad3-e286d4947718","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:17:55.738Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"aed7b903-e1f0-464b-90af-6c25cc18a1cc"}
{"level":"info","ts":"2026-07-13T16:17:55.738Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"aed7b903-e1f0-464b-90af-6c25cc18a1cc"}
{"level":"info","ts":"2026-07-13T16:17:55.738Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"aed7b903-e1f0-464b-90af-6c25cc18a1cc","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:17:55.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"aed7b903-e1f0-464b-90af-6c25cc18a1cc","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:17:55.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"aed7b903-e1f0-464b-90af-6c25cc18a1cc","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:17:55.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"aed7b903-e1f0-464b-90af-6c25cc18a1cc","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:17:55.739Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"aed7b903-e1f0-464b-90af-6c25cc18a1cc","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":0,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":0,"oldMaxWorkerReplicas":3,"newMaxWorkerReplicas":3}
{"level":"info","ts":"2026-07-13T16:17:55.739Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"aed7b903-e1f0-464b-90af-6c25cc18a1cc","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:17:55Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-07-13T16:17:55Z","stateTransitionTimes":{"ready":"2026-07-13T16:14:00Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.39","serviceIP":"10.134.0.39","podName":"ray-cluster-head-f82qs","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-07-13T16:13:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-07-13T16:14:00Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-07-13T16:13:30Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-07-13T16:17:55.750Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"e1b997e9-7726-49a6-8ae1-c23c1c00498d","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:17:55.750Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"2705ed3b-589a-488e-ac77-894efbe85659","rayCluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:17:55.750Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"2705ed3b-589a-488e-ac77-894efbe85659","pod":"ray-cluster-head-f82qs","ip":"10.134.0.39"}
{"level":"info","ts":"2026-07-13T16:17:55.750Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"2705ed3b-589a-488e-ac77-894efbe85659","pod":"ray-cluster-small-group-worker-2hvgl","ip":"10.133.0.41"}
{"level":"info","ts":"2026-07-13T16:17:55.751Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"2705ed3b-589a-488e-ac77-894efbe85659","rayCluster":"ray-cluster","podIPs":["10.134.0.39","10.133.0.41"]}
{"level":"info","ts":"2026-07-13T16:17:55.760Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"e1b997e9-7726-49a6-8ae1-c23c1c00498d","name":"ray-cluster-head"}
{"level":"info","ts":"2026-07-13T16:17:55.768Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"e1b997e9-7726-49a6-8ae1-c23c1c00498d","name":"ray-cluster-workers"}
{"level":"info","ts":"2026-07-13T16:17:55.768Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"namespace":"test-ns-4j8fw","name":"ray-cluster","reconcileID":"e1b997e9-7726-49a6-8ae1-c23c1c00498d","cluster":"ray-cluster"}
{"level":"info","ts":"2026-07-13T16:17:56.269Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"36cde0cc-f34b-4653-a143-b5403c2f8070"}
{"level":"info","ts":"2026-07-13T16:17:56.269Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"36cde0cc-f34b-4653-a143-b5403c2f8070"}
{"level":"info","ts":"2026-07-13T16:17:56.269Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"36cde0cc-f34b-4653-a143-b5403c2f8070","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:17:56.269Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"36cde0cc-f34b-4653-a143-b5403c2f8070","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:17:56.269Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"36cde0cc-f34b-4653-a143-b5403c2f8070","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:17:56.269Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"36cde0cc-f34b-4653-a143-b5403c2f8070","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-07-13T16:17:56.269Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"36cde0cc-f34b-4653-a143-b5403c2f8070","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:17:56.270Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"36cde0cc-f34b-4653-a143-b5403c2f8070","seconds":300}
{"level":"info","ts":"2026-07-13T16:17:56.272Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"efb0f6ed-a3e2-475c-92a5-5ef8298de05f"}
{"level":"info","ts":"2026-07-13T16:17:56.272Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"efb0f6ed-a3e2-475c-92a5-5ef8298de05f"}
{"level":"info","ts":"2026-07-13T16:17:56.272Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"efb0f6ed-a3e2-475c-92a5-5ef8298de05f","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-07-13T16:17:56.272Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"efb0f6ed-a3e2-475c-92a5-5ef8298de05f","Found 1 head Pod":"ray-cluster-head-f82qs","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-07-13T16:17:56.272Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"efb0f6ed-a3e2-475c-92a5-5ef8298de05f","head Pod":"ray-cluster-head-f82qs","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-f82qs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-07-13T16:17:56.272Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"efb0f6ed-a3e2-475c-92a5-5ef8298de05f","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-07-13T16:17:56.272Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"efb0f6ed-a3e2-475c-92a5-5ef8298de05f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-07-13T16:17:56.272Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"efb0f6ed-a3e2-475c-92a5-5ef8298de05f","namespace":"test-ns-4j8fw","name":"ray-cluster-small-group-worker-2hvgl"}
{"level":"info","ts":"2026-07-13T16:17:56.275Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"efb0f6ed-a3e2-475c-92a5-5ef8298de05f","The worker Pod has already been deleted":"ray-cluster-small-group-worker-2hvgl"}
{"level":"info","ts":"2026-07-13T16:17:56.275Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"efb0f6ed-a3e2-475c-92a5-5ef8298de05f","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-07-13T16:17:56.275Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"efb0f6ed-a3e2-475c-92a5-5ef8298de05f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-07-13T16:17:56.275Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"efb0f6ed-a3e2-475c-92a5-5ef8298de05f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-07-13T16:17:56.275Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4j8fw"},"reconcileID":"efb0f6ed-a3e2-475c-92a5-5ef8298de05f","seconds":300}